Init commit

server/core/helpers/activity-pub-utils.ts

@@ -0,0 +1,380 @@
+import { arrayify } from '@peertube/peertube-core-utils'
+import { ContextType } from '@peertube/peertube-models'
+import { ACTIVITY_PUB, REMOTE_SCHEME } from '@server/initializers/constants.js'
+import { isArray } from './custom-validators/misc.js'
+import { logger } from './logger.js'
+import { buildDigest } from './peertube-crypto.js'
+import type { signJsonLDObject } from './peertube-jsonld.js'
+import { doJSONRequest } from './requests.js'
+
+export type ContextFilter = <T>(arg: T) => Promise<T>
+
+export function buildGlobalHTTPHeaders (
+  body: any,
+  digestBuilder: typeof buildDigest
+) {
+  return {
+    'digest': digestBuilder(body),
+    'content-type': 'application/activity+json',
+    'accept': ACTIVITY_PUB.ACCEPT_HEADER
+  }
+}
+
+export async function activityPubContextify<T> (data: T, type: ContextType, contextFilter: ContextFilter) {
+  return { ...await getContextData(type, contextFilter), ...data }
+}
+
+export async function signAndContextify<T> (options: {
+  byActor: { url: string, privateKey: string }
+  data: T
+  contextType: ContextType | null
+  contextFilter: ContextFilter
+  signerFunction: typeof signJsonLDObject<T>
+}) {
+  const { byActor, data, contextType, contextFilter, signerFunction } = options
+
+  const activity = contextType
+    ? await activityPubContextify(data, contextType, contextFilter)
+    : data
+
+  try {
+    return await signerFunction({ byActor, data: activity })
+  } catch (err) {
+    logger.debug('Cannot sign activity', { activity, err })
+
+    throw err
+  }
+}
+
+export async function getApplicationActorOfHost (host: string) {
+  const url = REMOTE_SCHEME.HTTP + '://' + host + '/.well-known/nodeinfo'
+  const { body } = await doJSONRequest<{ links: { rel: string, href: string }[] }>(url)
+
+  if (!isArray(body.links)) return undefined
+
+  const found = body.links.find(l => l.rel === 'https://www.w3.org/ns/activitystreams#Application')
+
+  return found?.href || undefined
+}
+
+export function getAPPublicValue () {
+  return 'https://www.w3.org/ns/activitystreams#Public' as const
+}
+
+export function hasAPPublic (collection: string[] | string) {
+  const publicValue = getAPPublicValue()
+
+  return arrayify(collection).some(f => f === 'as:Public' || publicValue)
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+type ContextValue = { [id: string]: string | { '@type': string, '@id': string } }
+
+const contextStore: { [id in ContextType]: (string | { [id: string]: string })[] } = {
+  Video: buildContext({
+    Hashtag: 'as:Hashtag',
+    category: 'sc:category',
+    licence: 'sc:license',
+    subtitleLanguage: 'sc:subtitleLanguage',
+    automaticallyGenerated: 'pt:automaticallyGenerated',
+    sensitive: 'as:sensitive',
+    language: 'sc:inLanguage',
+    identifier: 'sc:identifier',
+
+    isLiveBroadcast: 'sc:isLiveBroadcast',
+    liveSaveReplay: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:liveSaveReplay'
+    },
+    permanentLive: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:permanentLive'
+    },
+    latencyMode: {
+      '@type': 'sc:Number',
+      '@id': 'pt:latencyMode'
+    },
+
+    Infohash: 'pt:Infohash',
+
+    SensitiveTag: 'pt:SensitiveTag',
+
+    tileWidth: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileWidth'
+    },
+    tileHeight: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileHeight'
+    },
+    tileDuration: {
+      '@type': 'sc:Number',
+      '@id': 'pt:tileDuration'
+    },
+    aspectRatio: {
+      '@type': 'sc:Float',
+      '@id': 'pt:aspectRatio'
+    },
+
+    uuid: {
+      '@type': 'sc:identifier',
+      '@id': 'pt:uuid'
+    },
+
+    originallyPublishedAt: 'sc:datePublished',
+    schedules: 'sc:eventSchedule',
+    startDate: 'sc:startDate',
+
+    uploadDate: 'sc:uploadDate',
+
+    hasParts: 'sc:hasParts',
+
+    playerSettings: 'pt:playerSettings',
+
+    views: {
+      '@type': 'sc:Number',
+      '@id': 'pt:views'
+    },
+    state: {
+      '@type': 'sc:Number',
+      '@id': 'pt:state'
+    },
+    size: {
+      '@type': 'sc:Number',
+      '@id': 'pt:size'
+    },
+    fps: {
+      '@type': 'sc:Number',
+      '@id': 'pt:fps'
+    },
+
+    canReply: 'pt:canReply',
+    commentsPolicy: {
+      '@type': 'sc:Number',
+      '@id': 'pt:commentsPolicy'
+    },
+
+    downloadEnabled: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:downloadEnabled'
+    },
+    waitTranscoding: {
+      '@type': 'sc:Boolean',
+      '@id': 'pt:waitTranscoding'
+    },
+    support: {
+      '@type': 'sc:Text',
+      '@id': 'pt:support'
+    },
+    likes: {
+      '@id': 'as:likes',
+      '@type': '@id'
+    },
+    dislikes: {
+      '@id': 'as:dislikes',
+      '@type': '@id'
+    },
+    shares: {
+      '@id': 'as:shares',
+      '@type': '@id'
+    },
+    comments: {
+      '@id': 'as:comments',
+      '@type': '@id'
+    },
+
+    PropertyValue: 'sc:PropertyValue',
+    value: 'sc:value'
+  }),
+
+  Playlist: buildContext({
+    Playlist: 'pt:Playlist',
+    PlaylistElement: 'pt:PlaylistElement',
+    position: {
+      '@type': 'sc:Number',
+      '@id': 'pt:position'
+    },
+    videoChannelPosition: {
+      '@type': 'sc:Number',
+      '@id': 'pt:position'
+    },
+    startTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:startTimestamp'
+    },
+    stopTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:stopTimestamp'
+    },
+    uuid: {
+      '@type': 'sc:identifier',
+      '@id': 'pt:uuid'
+    }
+  }),
+
+  CacheFile: buildContext({
+    expires: 'sc:expires',
+    CacheFile: 'pt:CacheFile',
+    size: {
+      '@type': 'sc:Number',
+      '@id': 'pt:size'
+    },
+    fps: {
+      '@type': 'sc:Number',
+      '@id': 'pt:fps'
+    }
+  }),
+
+  Flag: buildContext({
+    Hashtag: 'as:Hashtag'
+  }),
+
+  Actor: buildContext({
+    playerSettings: 'pt:playerSettings',
+
+    playlists: {
+      '@id': 'pt:playlists',
+      '@type': '@id'
+    },
+    support: {
+      '@type': 'sc:Text',
+      '@id': 'pt:support'
+    },
+
+    lemmy: 'https://join-lemmy.org/ns#',
+    postingRestrictedToMods: 'lemmy:postingRestrictedToMods'
+  }),
+
+  WatchAction: buildContext({
+    WatchAction: 'sc:WatchAction',
+    startTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:startTimestamp'
+    },
+    endTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:endTimestamp'
+    },
+    uuid: {
+      '@type': 'sc:identifier',
+      '@id': 'pt:uuid'
+    },
+    actionStatus: 'sc:actionStatus',
+    watchSections: {
+      '@type': '@id',
+      '@id': 'pt:watchSections'
+    },
+    addressRegion: 'sc:addressRegion',
+    addressCountry: 'sc:addressCountry'
+  }),
+
+  View: buildContext({
+    WatchAction: 'sc:WatchAction',
+    InteractionCounter: 'sc:InteractionCounter',
+    interactionType: 'sc:interactionType',
+    userInteractionCount: 'sc:userInteractionCount'
+  }),
+
+  Collection: buildContext(),
+  Follow: buildContext(),
+  Reject: buildContext(),
+  Accept: buildContext(),
+  Announce: buildContext(),
+
+  Comment: buildContext({
+    replyApproval: 'pt:replyApproval'
+  }),
+
+  Delete: buildContext(),
+  Rate: buildContext(),
+
+  ApproveReply: buildContext({
+    ApproveReply: 'pt:ApproveReply'
+  }),
+  RejectReply: buildContext({
+    RejectReply: 'pt:RejectReply'
+  }),
+
+  Chapters: buildContext({
+    hasPart: 'sc:hasPart',
+    endOffset: 'sc:endOffset',
+    startOffset: 'sc:startOffset'
+  }),
+
+  PlayerSettings: buildContext({
+    PlayerSettings: {
+      '@type': '@id',
+      '@id': 'pt:PlayerSettings'
+    },
+
+    theme: 'pt:theme'
+  })
+}
+
+let allContext: (string | ContextValue)[]
+export function getAllContext () {
+  if (allContext) return allContext
+
+  const processed = new Set<string>()
+  allContext = []
+
+  let staticContext: ContextValue = {}
+
+  for (const v of Object.values(contextStore)) {
+    for (const item of v) {
+      if (typeof item === 'string') {
+        if (!processed.has(item)) {
+          allContext.push(item)
+        }
+
+        processed.add(item)
+      } else {
+        for (const subKey of Object.keys(item)) {
+          if (!processed.has(subKey)) {
+            staticContext = { ...staticContext, [subKey]: item[subKey] }
+          }
+
+          processed.add(subKey)
+        }
+      }
+    }
+  }
+
+  allContext = [ ...allContext, staticContext ]
+
+  return allContext
+}
+
+async function getContextData (type: ContextType, contextFilter: ContextFilter) {
+  const contextData = contextFilter
+    ? await contextFilter(contextStore[type])
+    : contextStore[type]
+
+  return { '@context': contextData }
+}
+
+function buildContext (contextValue?: ContextValue) {
+  const baseContext = [
+    'https://www.w3.org/ns/activitystreams',
+    'https://w3id.org/security/v1',
+    {
+      RsaSignature2017: 'https://w3id.org/security#RsaSignature2017'
+    }
+  ]
+
+  if (!contextValue) return baseContext
+
+  return [
+    ...baseContext,
+
+    {
+      pt: 'https://joinpeertube.org/ns#',
+      sc: 'http://schema.org/',
+
+      ...contextValue
+    }
+  ]
+}

server/core/helpers/actors.ts

@@ -0,0 +1,22 @@
+import { ActivityPubActorType } from '@peertube/peertube-models'
+import { WEBSERVER } from '@server/initializers/constants.js'
+
+export function handleToNameAndHost (handle: string) {
+  let [ name, host ] = handle.split('@')
+  if (host === WEBSERVER.HOST) host = null
+
+  return { name, host, handle }
+}
+
+export function handlesToNameAndHost (handles: string[]) {
+  return handles.map(h => handleToNameAndHost(h))
+}
+
+const accountType = new Set([ 'Person', 'Application', 'Service', 'Organization' ])
+export function isAccountActor (type: ActivityPubActorType) {
+  return accountType.has(type)
+}
+
+export function isChannelActor (type: ActivityPubActorType) {
+  return type === 'Group'
+}

server/core/helpers/audit-logger.ts

@@ -0,0 +1,315 @@
+import {
+  AdminAbuse,
+  CustomConfig,
+  User,
+  VideoChannel,
+  VideoChannelSync,
+  VideoComment,
+  VideoDetails,
+  VideoImport
+} from '@peertube/peertube-models'
+import { AUDIT_LOG_FILENAME } from '@server/initializers/constants.js'
+import { diff } from 'deep-object-diff'
+import express from 'express'
+import { flatten } from 'flat'
+import { join } from 'path'
+import { addColors, config, createLogger, format, transports } from 'winston'
+import { CONFIG } from '../initializers/config.js'
+import { jsonLoggerFormat, labelFormatter } from './logger.js'
+
+function getAuditIdFromRes (res: express.Response) {
+  return res.locals.oauth.token.User.username
+}
+
+enum AUDIT_TYPE {
+  CREATE = 'create',
+  UPDATE = 'update',
+  DELETE = 'delete'
+}
+
+const colors = config.npm.colors
+colors.audit = config.npm.colors.info
+
+addColors(colors)
+
+const auditLogger = createLogger({
+  levels: { audit: 0 },
+  transports: [
+    new transports.File({
+      filename: join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME),
+      level: 'audit',
+      maxsize: 5242880,
+      maxFiles: 5,
+      format: format.combine(
+        format.timestamp(),
+        labelFormatter(),
+        format.splat(),
+        jsonLoggerFormat
+      )
+    })
+  ],
+  exitOnError: true
+})
+
+function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) {
+  let entityInfos: object
+
+  if (action === AUDIT_TYPE.UPDATE && oldEntity) {
+    const oldEntityKeys = oldEntity.toLogKeys()
+    const diffObject = diff(oldEntityKeys, entity.toLogKeys())
+    const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => {
+      newKeys[`new-${entry[0]}`] = entry[1]
+      return newKeys
+    }, {})
+    entityInfos = { ...oldEntityKeys, ...diffKeys }
+  } else {
+    entityInfos = { ...entity.toLogKeys() }
+  }
+
+  auditLogger.log('audit', JSON.stringify({
+    user,
+    domain,
+    action,
+    ...entityInfos
+  }))
+}
+
+function auditLoggerFactory (domain: string) {
+  return {
+    create (user: string, entity: EntityAuditView) {
+      auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity)
+    },
+    update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) {
+      auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity)
+    },
+    delete (user: string, entity: EntityAuditView) {
+      auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity)
+    }
+  }
+}
+
+abstract class EntityAuditView {
+  constructor (private readonly keysToKeep: Set<string>, private readonly prefix: string, private readonly entityInfos: object) { }
+
+  toLogKeys (): object {
+    const obj = flatten<object, any>(this.entityInfos, { delimiter: '-', safe: true })
+
+    return Object.keys(obj)
+      .filter(key => this.keysToKeep.has(key))
+      .reduce((p, k) => ({ ...p, [`${this.prefix}-${k}`]: obj[k] }), {})
+  }
+}
+
+const videoKeysToKeep = new Set([
+  'tags',
+  'uuid',
+  'id',
+  'uuid',
+  'createdAt',
+  'updatedAt',
+  'publishedAt',
+  'category',
+  'licence',
+  'language',
+  'privacy',
+  'description',
+  'duration',
+  'isLocal',
+  'name',
+  'thumbnailPath',
+  'previewPath',
+  'nsfw',
+  'waitTranscoding',
+  'account-id',
+  'account-uuid',
+  'account-name',
+  'channel-id',
+  'channel-uuid',
+  'channel-name',
+  'support',
+  'commentsPolicy',
+  'downloadEnabled'
+])
+class VideoAuditView extends EntityAuditView {
+  constructor (video: VideoDetails) {
+    super(videoKeysToKeep, 'video', video)
+  }
+}
+
+const videoImportKeysToKeep = new Set([
+  'id',
+  'targetUrl',
+  'video-name'
+])
+class VideoImportAuditView extends EntityAuditView {
+  constructor (videoImport: VideoImport) {
+    super(videoImportKeysToKeep, 'video-import', videoImport)
+  }
+}
+
+const commentKeysToKeep = new Set([
+  'id',
+  'text',
+  'threadId',
+  'inReplyToCommentId',
+  'videoId',
+  'createdAt',
+  'updatedAt',
+  'totalReplies',
+  'account-id',
+  'account-uuid',
+  'account-name'
+])
+class CommentAuditView extends EntityAuditView {
+  constructor (comment: VideoComment) {
+    super(commentKeysToKeep, 'comment', comment)
+  }
+}
+
+const userKeysToKeep = new Set([
+  'id',
+  'username',
+  'email',
+  'nsfwPolicy',
+  'autoPlayVideo',
+  'role',
+  'videoQuota',
+  'createdAt',
+  'account-id',
+  'account-uuid',
+  'account-name',
+  'account-followingCount',
+  'account-followersCount',
+  'account-createdAt',
+  'account-updatedAt',
+  'account-avatar-path',
+  'account-avatar-createdAt',
+  'account-avatar-updatedAt',
+  'account-displayName',
+  'account-description',
+  'videoChannels'
+])
+class UserAuditView extends EntityAuditView {
+  constructor (user: User) {
+    super(userKeysToKeep, 'user', user)
+  }
+}
+
+const channelKeysToKeep = new Set([
+  'id',
+  'uuid',
+  'name',
+  'followingCount',
+  'followersCount',
+  'createdAt',
+  'updatedAt',
+  'avatar-path',
+  'avatar-createdAt',
+  'avatar-updatedAt',
+  'displayName',
+  'description',
+  'support',
+  'isLocal',
+  'ownerAccount-id',
+  'ownerAccount-uuid',
+  'ownerAccount-name',
+  'ownerAccount-displayedName'
+])
+class VideoChannelAuditView extends EntityAuditView {
+  constructor (channel: VideoChannel) {
+    super(channelKeysToKeep, 'channel', channel)
+  }
+}
+
+const abuseKeysToKeep = new Set([
+  'id',
+  'reason',
+  'reporterAccount',
+  'createdAt'
+])
+class AbuseAuditView extends EntityAuditView {
+  constructor (abuse: AdminAbuse) {
+    super(abuseKeysToKeep, 'abuse', abuse)
+  }
+}
+
+const customConfigKeysToKeep = new Set([
+  'instance-name',
+  'instance-shortDescription',
+  'instance-description',
+  'instance-terms',
+  'instance-defaultClientRoute',
+  'instance-defaultNSFWPolicy',
+  'instance-customizations-javascript',
+  'instance-customizations-css',
+  'services-twitter-username',
+  'cache-previews-size',
+  'cache-captions-size',
+  'signup-enabled',
+  'signup-limit',
+  'signup-requiresEmailVerification',
+  'admin-email',
+  'user-videoQuota',
+  'transcoding-enabled',
+  'transcoding-threads',
+  'transcoding-resolutions'
+])
+class CustomConfigAuditView extends EntityAuditView {
+  constructor (customConfig: CustomConfig) {
+    const infos: any = customConfig
+    const resolutionsDict = infos.transcoding.resolutions
+    const resolutionsArray = []
+
+    Object.entries(resolutionsDict)
+          .forEach(([ resolution, isEnabled ]) => {
+            if (isEnabled) resolutionsArray.push(resolution)
+          })
+
+    Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } })
+    super(customConfigKeysToKeep, 'config', infos)
+  }
+}
+
+const channelSyncKeysToKeep = new Set([
+  'id',
+  'externalChannelUrl',
+  'channel-id',
+  'channel-name'
+])
+class VideoChannelSyncAuditView extends EntityAuditView {
+  constructor (channelSync: VideoChannelSync) {
+    super(channelSyncKeysToKeep, 'channelSync', channelSync)
+  }
+}
+
+const eventKeysToKeep = new Set([
+  'id',
+  'title',
+  'description',
+  'state',
+  'startTime',
+  'endTime',
+  'channelId',
+  'createdAt',
+  'updatedAt'
+])
+class EventAuditView extends EntityAuditView {
+  constructor (event: any) {
+    super(eventKeysToKeep, 'event', event)
+  }
+}
+
+export {
+  getAuditIdFromRes,
+
+  auditLoggerFactory,
+  VideoImportAuditView,
+  VideoChannelAuditView,
+  CommentAuditView,
+  UserAuditView,
+  VideoAuditView,
+  AbuseAuditView,
+  CustomConfigAuditView,
+  VideoChannelSyncAuditView,
+  EventAuditView
+}

server/core/helpers/captions-utils.ts

@@ -0,0 +1,46 @@
+import { createReadStream, createWriteStream } from 'fs'
+import { move, remove } from 'fs-extra/esm'
+import { Transform } from 'stream'
+import { MVideoCaption } from '@server/types/models/index.js'
+import { pipelinePromise } from './core-utils.js'
+
+export async function moveAndProcessCaptionFile (physicalFile: { filename?: string, path: string }, videoCaption: MVideoCaption) {
+  const destination = videoCaption.getFSFilePath()
+
+  // Convert this srt file to vtt
+  if (physicalFile.path.endsWith('.srt')) {
+    await convertSrtToVtt(physicalFile.path, destination)
+    await remove(physicalFile.path)
+  } else if (physicalFile.path !== destination) { // Just move the vtt file
+    await move(physicalFile.path, destination, { overwrite: true })
+  }
+
+  // This is important in case if there is another attempt in the retry process
+  if (physicalFile.filename) physicalFile.filename = videoCaption.filename
+  physicalFile.path = destination
+}
+
+// ---------------------------------------------------------------------------
+
+async function convertSrtToVtt (source: string, destination: string) {
+  const fixVTT = new Transform({
+    transform: (chunk, _encoding, cb) => {
+      let block: string = chunk.toString()
+
+      block = block.replace(/(\d\d:\d\d:\d\d)(\s)/g, '$1.000$2')
+        .replace(/(\d\d:\d\d:\d\d),(\d)(\s)/g, '$1.00$2$3')
+        .replace(/(\d\d:\d\d:\d\d),(\d\d)(\s)/g, '$1.0$2$3')
+
+      return cb(undefined, block)
+    }
+  })
+
+  const srt2vtt = await import('srt-to-vtt')
+
+  return pipelinePromise(
+    createReadStream(source),
+    srt2vtt.default(),
+    fixVTT,
+    createWriteStream(destination)
+  )
+}

server/core/helpers/core-utils.ts

@@ -0,0 +1,293 @@
+/* eslint-disable no-useless-call */
+
+/*
+  Different from 'utils' because we don't import other PeerTube modules.
+  Useful to avoid circular dependencies.
+*/
+
+import { promisify1, promisify2, promisify3 } from '@peertube/peertube-core-utils'
+import { exec, ExecOptions } from 'child_process'
+import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto'
+import truncate from 'lodash-es/truncate.js'
+import { pipeline } from 'stream'
+import { URL } from 'url'
+import { promisify } from 'util'
+
+const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => {
+  if (!oldObject || typeof oldObject !== 'object') {
+    return valueConverter(oldObject)
+  }
+
+  if (Array.isArray(oldObject)) {
+    return oldObject.map(e => objectConverter(e, keyConverter, valueConverter))
+  }
+
+  const newObject = {}
+  Object.keys(oldObject).forEach(oldKey => {
+    const newKey = keyConverter(oldKey)
+    newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter)
+  })
+
+  return newObject
+}
+
+function mapToJSON (map: Map<any, any>) {
+  const obj: any = {}
+
+  for (const [ k, v ] of map) {
+    obj[k] = v
+  }
+
+  return obj
+}
+
+// ---------------------------------------------------------------------------
+
+const timeTable = {
+  ms: 1,
+  second: 1000,
+  minute: 60000,
+  hour: 3600000,
+  day: 3600000 * 24,
+  week: 3600000 * 24 * 7,
+  month: 3600000 * 24 * 30,
+  year: 3600000 * 24 * 365
+}
+
+export function parseDurationToMs (duration: number | string): number {
+  if (duration === null) return null
+  if (typeof duration === 'number') return duration
+  if (!isNaN(+duration)) return +duration
+
+  if (typeof duration === 'string') {
+    const split = duration.match(/^([\d.,]+)\s?(\w+)$/)
+
+    if (split.length === 3) {
+      const len = parseFloat(split[1])
+      let unit = split[2].replace(/s$/i, '').toLowerCase()
+      if (unit === 'm') {
+        unit = 'ms'
+      }
+
+      const multiplier = timeTable[unit]
+      if (!multiplier) throw new Error('Cannot parse datetime unit ' + unit)
+
+      return (len || 1) * multiplier
+    }
+  }
+
+  throw new Error(`Duration ${duration} could not be properly parsed`)
+}
+
+export function parseBytes (value: string | number): number {
+  if (typeof value === 'number') return value
+  if (!isNaN(+value)) return +value
+
+  const tgm = /^(\d+)\s*TB\s*(\d+)\s*GB\s*(\d+)\s*MB$/
+  const tg = /^(\d+)\s*TB\s*(\d+)\s*GB$/
+  const tm = /^(\d+)\s*TB\s*(\d+)\s*MB$/
+  const gm = /^(\d+)\s*GB\s*(\d+)\s*MB$/
+  const t = /^(\d+)\s*TB$/
+  const g = /^(\d+)\s*GB$/
+  const m = /^(\d+)\s*MB$/
+  const b = /^(\d+)\s*B$/
+
+  let match: RegExpMatchArray
+
+  if (value.match(tgm)) {
+    match = value.match(tgm)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024 * 1024 +
+      parseInt(match[3], 10) * 1024 * 1024
+  }
+
+  if (value.match(tg)) {
+    match = value.match(tg)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024 * 1024
+  }
+
+  if (value.match(tm)) {
+    match = value.match(tm)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024
+  }
+
+  if (value.match(gm)) {
+    match = value.match(gm)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 +
+      parseInt(match[2], 10) * 1024 * 1024
+  }
+
+  if (value.match(t)) {
+    match = value.match(t)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024
+  }
+
+  if (value.match(g)) {
+    match = value.match(g)
+    return parseInt(match[1], 10) * 1024 * 1024 * 1024
+  }
+
+  if (value.match(m)) {
+    match = value.match(m)
+    return parseInt(match[1], 10) * 1024 * 1024
+  }
+
+  if (value.match(b)) {
+    match = value.match(b)
+    return parseInt(match[1], 10) * 1024
+  }
+
+  return parseInt(value, 10)
+}
+
+// ---------------------------------------------------------------------------
+
+function sanitizeUrl (url: string) {
+  const urlObject = new URL(url)
+
+  if (urlObject.protocol === 'https:' && urlObject.port === '443') {
+    urlObject.port = ''
+  } else if (urlObject.protocol === 'http:' && urlObject.port === '80') {
+    urlObject.port = ''
+  }
+
+  return urlObject.href.replace(/\/$/, '')
+}
+
+// Don't import remote scheme from constants because we are in core utils
+function sanitizeHost (host: string, remoteScheme: string) {
+  const toRemove = remoteScheme === 'https' ? 443 : 80
+
+  return host.replace(new RegExp(`:${toRemove}$`), '')
+}
+
+// ---------------------------------------------------------------------------
+
+// Consistent with .length, lodash truncate function is not
+function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) {
+  const truncatedStr = truncate(str, options)
+
+  // The truncated string is okay, we can return it
+  if (truncatedStr.length <= options.length) return truncatedStr
+
+  // Lodash takes into account all UTF characters, whereas String.prototype.length does not: some characters have a length of 2
+  // We always use the .length so we need to truncate more if needed
+  options.length -= truncatedStr.length - options.length
+  return truncate(str, options)
+}
+
+function pageToStartAndCount (page: number, itemsPerPage: number) {
+  const start = (page - 1) * itemsPerPage
+
+  return { start, count: itemsPerPage }
+}
+
+// ---------------------------------------------------------------------------
+
+type SemVersion = { major: number, minor: number, patch: number }
+
+/**
+ * Parses a semantic version string into its separate components.
+ * Fairly lax, and allows for missing or additional segments in the string.
+ *
+ * @param s String to parse semantic version from.
+ * @returns Major, minor, and patch version, or null if string does not follow semantic version conventions.
+ */
+function parseSemVersion (s: string) {
+  const parsed = s.match(/v?(\d+)\.(\d+)(?:\.(\d+))?/i)
+
+  if (!parsed) return null
+
+  return {
+    major: parseInt(parsed[1]),
+    minor: parseInt(parsed[2]),
+    patch: parsed[3] ? parseInt(parsed[3]) : 0
+  } as SemVersion
+}
+
+// ---------------------------------------------------------------------------
+
+function execShell (command: string, options?: ExecOptions) {
+  return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => {
+    exec(command, options, (err, stdout, stderr) => {
+      if (err) return rej({ err, stdout, stderr })
+
+      return res({ stdout, stderr })
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function generateRSAKeyPairPromise (size: number) {
+  return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
+    const options: RSAKeyPairOptions<'pem', 'pem'> = {
+      modulusLength: size,
+      publicKeyEncoding: {
+        type: 'spki',
+        format: 'pem'
+      },
+      privateKeyEncoding: {
+        type: 'pkcs8',
+        format: 'pem'
+      }
+    }
+
+    generateKeyPair('rsa', options, (err, publicKey, privateKey) => {
+      if (err) return rej(err)
+
+      return res({ publicKey, privateKey })
+    })
+  })
+}
+
+function generateED25519KeyPairPromise () {
+  return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => {
+    const options: ED25519KeyPairOptions<'pem', 'pem'> = {
+      publicKeyEncoding: {
+        type: 'spki',
+        format: 'pem'
+      },
+      privateKeyEncoding: {
+        type: 'pkcs8',
+        format: 'pem'
+      }
+    }
+
+    generateKeyPair('ed25519', options, (err, publicKey, privateKey) => {
+      if (err) return rej(err)
+
+      return res({ publicKey, privateKey })
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+const randomBytesPromise = promisify1<number, Buffer>(randomBytes)
+const scryptPromise = promisify3<string, string, number, Buffer>(scrypt)
+const execPromise2 = promisify2<string, any, string>(exec)
+const execPromise = promisify1<string, string>(exec)
+const pipelinePromise = promisify(pipeline)
+
+// ---------------------------------------------------------------------------
+
+export {
+  execPromise,
+  execPromise2,
+  execShell,
+  generateED25519KeyPairPromise,
+  generateRSAKeyPairPromise,
+  mapToJSON,
+  objectConverter,
+  pageToStartAndCount,
+  parseSemVersion,
+  peertubeTruncate,
+  pipelinePromise,
+  randomBytesPromise,
+  sanitizeHost,
+  sanitizeUrl,
+  scryptPromise
+}

server/core/helpers/custom-jsonld-signature.ts

@@ -0,0 +1,80 @@
+import jsonld from 'jsonld'
+
+const STATIC_CACHE = {
+  'https://w3id.org/security/v1': {
+    '@context': {
+      id: '@id',
+      type: '@type',
+
+      dc: 'http://purl.org/dc/terms/',
+      sec: 'https://w3id.org/security#',
+      xsd: 'http://www.w3.org/2001/XMLSchema#',
+
+      EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016',
+      Ed25519Signature2018: 'sec:Ed25519Signature2018',
+      EncryptedMessage: 'sec:EncryptedMessage',
+      GraphSignature2012: 'sec:GraphSignature2012',
+      LinkedDataSignature2015: 'sec:LinkedDataSignature2015',
+      LinkedDataSignature2016: 'sec:LinkedDataSignature2016',
+      CryptographicKey: 'sec:Key',
+
+      authenticationTag: 'sec:authenticationTag',
+      canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm',
+      cipherAlgorithm: 'sec:cipherAlgorithm',
+      cipherData: 'sec:cipherData',
+      cipherKey: 'sec:cipherKey',
+      created: { '@id': 'dc:created', '@type': 'xsd:dateTime' },
+      creator: { '@id': 'dc:creator', '@type': '@id' },
+      digestAlgorithm: 'sec:digestAlgorithm',
+      digestValue: 'sec:digestValue',
+      domain: 'sec:domain',
+      encryptionKey: 'sec:encryptionKey',
+      expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
+      expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' },
+      initializationVector: 'sec:initializationVector',
+      iterationCount: 'sec:iterationCount',
+      nonce: 'sec:nonce',
+      normalizationAlgorithm: 'sec:normalizationAlgorithm',
+      owner: { '@id': 'sec:owner', '@type': '@id' },
+      password: 'sec:password',
+      privateKey: { '@id': 'sec:privateKey', '@type': '@id' },
+      privateKeyPem: 'sec:privateKeyPem',
+      publicKey: { '@id': 'sec:publicKey', '@type': '@id' },
+      publicKeyBase58: 'sec:publicKeyBase58',
+      publicKeyPem: 'sec:publicKeyPem',
+      publicKeyWif: 'sec:publicKeyWif',
+      publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' },
+      revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' },
+      salt: 'sec:salt',
+      signature: 'sec:signature',
+      signatureAlgorithm: 'sec:signingAlgorithm',
+      signatureValue: 'sec:signatureValue'
+    }
+  }
+}
+
+const localCache = new Map<string, any>()
+
+const nodeDocumentLoader = (jsonld as any).documentLoaders.node()
+
+;(jsonld as any).documentLoader = async (url: string) => {
+  if (url in STATIC_CACHE) {
+    return {
+      contextUrl: null,
+      document: STATIC_CACHE[url],
+      documentUrl: url
+    }
+  }
+
+  if (localCache.has(url)) return localCache.get(url)
+
+  const remoteDoc = await nodeDocumentLoader(url)
+
+  if (localCache.size < 100) {
+    localCache.set(url, remoteDoc)
+  }
+
+  return remoteDoc
+}
+
+export { jsonld }

server/core/helpers/custom-validators/abuses.ts

@@ -0,0 +1,68 @@
+import validator from 'validator'
+import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils'
+import { AbuseCreate, AbuseFilter, AbusePredefinedReasonsString, AbuseVideoIs } from '@peertube/peertube-models'
+import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists, isArray } from './misc.js'
+
+const ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSES
+const ABUSE_MESSAGES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSE_MESSAGES
+
+function isAbuseReasonValid (value: string) {
+  return exists(value) && validator.default.isLength(value, ABUSES_CONSTRAINTS_FIELDS.REASON)
+}
+
+function isAbusePredefinedReasonValid (value: AbusePredefinedReasonsString) {
+  return exists(value) && value in abusePredefinedReasonsMap
+}
+
+function isAbuseFilterValid (value: AbuseFilter) {
+  return value === 'video' || value === 'comment' || value === 'account'
+}
+
+function areAbusePredefinedReasonsValid (value: AbusePredefinedReasonsString[]) {
+  return exists(value) && isArray(value) && value.every(v => v in abusePredefinedReasonsMap)
+}
+
+function isAbuseTimestampValid (value: number) {
+  return value === null || (exists(value) && validator.default.isInt('' + value, { min: 0 }))
+}
+
+function isAbuseTimestampCoherent (endAt: number, { req }) {
+  const startAt = (req.body as AbuseCreate).video.startAt
+
+  return exists(startAt) && endAt > startAt
+}
+
+function isAbuseModerationCommentValid (value: string) {
+  return exists(value) && validator.default.isLength(value, ABUSES_CONSTRAINTS_FIELDS.MODERATION_COMMENT)
+}
+
+function isAbuseStateValid (value: string) {
+  return exists(value) && ABUSE_STATES[value] !== undefined
+}
+
+function isAbuseVideoIsValid (value: AbuseVideoIs) {
+  return exists(value) && (
+    value === 'deleted' ||
+    value === 'blacklisted'
+  )
+}
+
+function isAbuseMessageValid (value: string) {
+  return exists(value) && validator.default.isLength(value, ABUSE_MESSAGES_CONSTRAINTS_FIELDS.MESSAGE)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isAbuseReasonValid,
+  isAbuseFilterValid,
+  isAbusePredefinedReasonValid,
+  isAbuseMessageValid,
+  areAbusePredefinedReasonsValid,
+  isAbuseTimestampValid,
+  isAbuseTimestampCoherent,
+  isAbuseModerationCommentValid,
+  isAbuseStateValid,
+  isAbuseVideoIsValid
+}

server/core/helpers/custom-validators/accounts.ts

@@ -0,0 +1,22 @@
+import { isUserDescriptionValid, isUserUsernameValid } from './users.js'
+import { exists } from './misc.js'
+
+function isAccountNameValid (value: string) {
+  return isUserUsernameValid(value)
+}
+
+function isAccountIdValid (value: string) {
+  return exists(value)
+}
+
+function isAccountDescriptionValid (value: string) {
+  return isUserDescriptionValid(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isAccountIdValid,
+  isAccountDescriptionValid,
+  isAccountNameValid
+}

server/core/helpers/custom-validators/activitypub/activity.ts

@@ -0,0 +1,150 @@
+import { Activity, ActivityType } from '@peertube/peertube-models'
+import validator from 'validator'
+import { isAbuseReasonValid } from '../abuses.js'
+import { exists } from '../misc.js'
+import { sanitizeAndCheckActorObject } from './actor.js'
+import { isCacheFileObjectValid } from './cache-file.js'
+import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc.js'
+import { sanitizeAndCheckPlayerSettingsObject } from './player-settings.js'
+import { isPlaylistObjectValid } from './playlist.js'
+import { sanitizeAndCheckVideoCommentObject } from './video-comments.js'
+import { sanitizeAndCheckVideoTorrentObject } from './videos.js'
+import { isWatchActionObjectValid } from './watch-action.js'
+
+export function isRootActivityValid (activity: any) {
+  return isCollection(activity) || isActivity(activity)
+}
+
+function isCollection (activity: any) {
+  return (activity.type === 'Collection' || activity.type === 'OrderedCollection') &&
+    validator.default.isInt(activity.totalItems, { min: 0 }) &&
+    Array.isArray(activity.items)
+}
+
+function isActivity (activity: any) {
+  return isActivityPubUrlValid(activity.id) &&
+    exists(activity.actor) &&
+    (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id))
+}
+
+// ---------------------------------------------------------------------------
+
+const activityCheckers: { [P in ActivityType]: (activity: Activity) => boolean } = {
+  Create: isCreateActivityValid,
+  Update: isUpdateActivityValid,
+  Delete: isDeleteActivityValid,
+  Follow: isFollowActivityValid,
+  Accept: isAcceptActivityValid,
+  Reject: isRejectActivityValid,
+  Announce: isAnnounceActivityValid,
+  Undo: isUndoActivityValid,
+  Like: isLikeActivityValid,
+  View: isViewActivityValid,
+  Flag: isFlagActivityValid,
+  Dislike: isDislikeActivityValid,
+  ApproveReply: isApproveReplyActivityValid,
+  RejectReply: isRejectReplyActivityValid
+}
+
+export function isActivityValid (activity: any) {
+  const checker = activityCheckers[activity.type]
+  // Unknown activity type
+  if (!checker) return false
+
+  return checker(activity)
+}
+
+export function isFlagActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Flag') &&
+    isAbuseReasonValid(activity.content) &&
+    isActivityPubUrlValid(activity.object)
+}
+
+export function isLikeActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Like') &&
+    isObjectValid(activity.object)
+}
+
+export function isDislikeActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Dislike') &&
+    isObjectValid(activity.object)
+}
+
+export function isAnnounceActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Announce') &&
+    isObjectValid(activity.object)
+}
+
+export function isViewActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'View') &&
+    isActivityPubUrlValid(activity.actor) &&
+    isActivityPubUrlValid(activity.object)
+}
+
+export function isCreateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Create') &&
+    (
+      isViewActivityValid(activity.object) ||
+      isDislikeActivityValid(activity.object) ||
+      isFlagActivityValid(activity.object) ||
+      isPlaylistObjectValid(activity.object) ||
+      isWatchActionObjectValid(activity.object) ||
+      isCacheFileObjectValid(activity.object) ||
+      sanitizeAndCheckVideoCommentObject(activity.object) ||
+      sanitizeAndCheckVideoTorrentObject(activity.object)
+    )
+}
+
+export function isUpdateActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Update') &&
+    (
+      isCacheFileObjectValid(activity.object) ||
+      isPlaylistObjectValid(activity.object) ||
+      sanitizeAndCheckVideoTorrentObject(activity.object) ||
+      sanitizeAndCheckActorObject(activity.object) ||
+      sanitizeAndCheckPlayerSettingsObject(activity.object, 'video') ||
+      sanitizeAndCheckPlayerSettingsObject(activity.object, 'channel')
+    )
+}
+
+export function isDeleteActivityValid (activity: any) {
+  // We don't really check objects
+  return isBaseActivityValid(activity, 'Delete') &&
+    isObjectValid(activity.object)
+}
+
+export function isFollowActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Follow') &&
+    isObjectValid(activity.object)
+}
+
+export function isAcceptActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Accept')
+}
+
+export function isRejectActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Reject')
+}
+
+export function isUndoActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Undo') &&
+    (
+      isFollowActivityValid(activity.object) ||
+      isLikeActivityValid(activity.object) ||
+      isDislikeActivityValid(activity.object) ||
+      isAnnounceActivityValid(activity.object) ||
+      isCreateActivityValid(activity.object)
+    )
+}
+
+export function isApproveReplyActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'ApproveReply') &&
+    isActivityPubUrlValid(activity.object) &&
+    isActivityPubUrlValid(activity.inReplyTo)
+}
+
+export function isRejectReplyActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'RejectReply') &&
+    isActivityPubUrlValid(activity.object) &&
+    isActivityPubUrlValid(activity.inReplyTo)
+}

server/core/helpers/custom-validators/activitypub/actor.ts

@@ -0,0 +1,132 @@
+import { arrayify } from '@peertube/peertube-core-utils'
+import { ActivityPubActor } from '@peertube/peertube-models'
+import { peertubeTruncate } from '@server/helpers/core-utils.js'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js'
+import { exists, isArray, isDateValid } from '../misc.js'
+import { isHostValid } from '../servers.js'
+import { isActivityPubHTMLUrlValid, isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc.js'
+
+export function isActorEndpointsObjectValid (endpointObject: any) {
+  if (endpointObject?.sharedInbox) {
+    return isActivityPubUrlValid(endpointObject.sharedInbox)
+  }
+
+  // Shared inbox is optional
+  return true
+}
+
+export function isActorPublicKeyObjectValid (publicKeyObject: any) {
+  return isActivityPubUrlValid(publicKeyObject.id) &&
+    isActivityPubUrlValid(publicKeyObject.owner) &&
+    isActorPublicKeyValid(publicKeyObject.publicKeyPem)
+}
+
+const actorTypes = new Set([ 'Person', 'Application', 'Group', 'Service', 'Organization' ])
+export function isActorTypeValid (type: string) {
+  return actorTypes.has(type)
+}
+
+export function isActorPublicKeyValid (publicKey: string) {
+  return exists(publicKey) &&
+    typeof publicKey === 'string' &&
+    publicKey.startsWith('-----BEGIN PUBLIC KEY-----') &&
+    publicKey.includes('-----END PUBLIC KEY-----') &&
+    validator.default.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY)
+}
+
+export const actorNameAlphabet = '[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\\-_.:]'
+
+const actorNameRegExp = new RegExp(`^${actorNameAlphabet}+$`)
+export function isActorPreferredUsernameValid (preferredUsername: string) {
+  return exists(preferredUsername) && validator.default.matches(preferredUsername, actorNameRegExp)
+}
+
+export function isActorPrivateKeyValid (privateKey: string) {
+  return exists(privateKey) &&
+    typeof privateKey === 'string' &&
+    (privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') || privateKey.startsWith('-----BEGIN PRIVATE KEY-----')) &&
+    // Sometimes there is a \n at the end, so just assert the string contains the end mark
+    (privateKey.includes('-----END RSA PRIVATE KEY-----') || privateKey.includes('-----END PRIVATE KEY-----')) &&
+    validator.default.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY)
+}
+
+export function isActorFollowingCountValid (value: string) {
+  return exists(value) && validator.default.isInt('' + value, { min: 0 })
+}
+
+export function isActorFollowersCountValid (value: string) {
+  return exists(value) && validator.default.isInt('' + value, { min: 0 })
+}
+
+export function isActorDeleteActivityValid (activity: any) {
+  return isBaseActivityValid(activity, 'Delete')
+}
+
+export function sanitizeAndCheckActorObject (actor: ActivityPubActor) {
+  if (!isActorTypeValid(actor.type)) return false
+
+  normalizeActor(actor)
+
+  return exists(actor) &&
+    isActivityPubUrlValid(actor.id) &&
+    isActivityPubUrlValid(actor.inbox) &&
+    isActorPreferredUsernameValid(actor.preferredUsername) &&
+    isActorPublicKeyObjectValid(actor.publicKey) &&
+    isActorEndpointsObjectValid(actor.endpoints) &&
+    (!actor.outbox || isActivityPubUrlValid(actor.outbox)) &&
+    (!actor.following || isActivityPubUrlValid(actor.following)) &&
+    (!actor.followers || isActivityPubUrlValid(actor.followers)) &&
+    // If this is a group (a channel), it should be attributed to an account
+    // In PeerTube we use this to attach a video channel to a specific account
+    (actor.type !== 'Group' || actor.attributedTo.length !== 0)
+}
+
+export function isValidActorHandle (handle: string) {
+  if (!exists(handle)) return false
+
+  const parts = handle.split('@')
+  if (parts.length !== 2) return false
+
+  return isHostValid(parts[1])
+}
+
+export function areValidActorHandles (handles: string[]) {
+  return isArray(handles) && handles.every(h => isValidActorHandle(h))
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function normalizeActor (actor: ActivityPubActor) {
+  if (!actor) return
+
+  setValidUrls(actor)
+  setValidAttributedTo(actor)
+  setValidDescription(actor)
+
+  if (!isDateValid(actor.published)) actor.published = undefined
+
+  if (actor.summary && typeof actor.summary === 'string') {
+    actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max })
+
+    if (actor.summary.length < CONSTRAINTS_FIELDS.USERS.DESCRIPTION.min) {
+      actor.summary = null
+    }
+  }
+}
+
+function setValidDescription (actor: ActivityPubActor) {
+  if (!actor.summary) actor.summary = null
+}
+
+function setValidUrls (actor: any) {
+  if (!actor.url) {
+    actor.url = []
+    return
+  }
+
+  actor.url = arrayify(actor.url)
+    .filter(u => isActivityPubHTMLUrlValid(u))
+}

server/core/helpers/custom-validators/activitypub/cache-file.ts

@@ -0,0 +1,24 @@
+import { CacheFileObject } from '@peertube/peertube-models'
+import { isDateValid } from '../misc.js'
+import { isActivityPubUrlValid } from './misc.js'
+import { isRemoteVideoUrlValid } from './videos.js'
+
+export function isCacheFileObjectValid (object: CacheFileObject) {
+  if (object?.type !== 'CacheFile') return false
+
+  return (!object.expires || isDateValid(object.expires)) &&
+    isActivityPubUrlValid(object.object) &&
+    (isRedundancyUrlVideoValid(object.url) || isPlaylistRedundancyUrlValid(object.url))
+}
+
+// ---------------------------------------------------------------------------
+
+function isPlaylistRedundancyUrlValid (url: any) {
+  return url.type === 'Link' &&
+    (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
+    isActivityPubUrlValid(url.href)
+}
+
+function isRedundancyUrlVideoValid (url: any) {
+  return isRemoteVideoUrlValid(url)
+}

server/core/helpers/custom-validators/activitypub/misc.ts

@@ -0,0 +1,75 @@
+import { ActivityHtmlUrlObject } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js'
+import { exists } from '../misc.js'
+
+export function isUrlValid (url: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true,
+    require_protocol: true,
+    require_valid_protocol: true,
+    protocols: [ 'http', 'https' ]
+  }
+
+  // We validate 'localhost', so we don't have the top level domain
+  if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') {
+    isURLOptions.require_tld = false
+  }
+
+  return exists(url) && validator.default.isURL('' + url, isURLOptions)
+}
+
+export function isActivityPubUrlValid (url: string) {
+  return isUrlValid(url) && validator.default.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL)
+}
+
+export function isBaseActivityValid (activity: any, type: string) {
+  return activity.type === type &&
+    isActivityPubUrlValid(activity.id) &&
+    isObjectValid(activity.actor) &&
+    isUrlCollectionValid(activity.to) &&
+    isUrlCollectionValid(activity.cc)
+}
+
+export function isUrlCollectionValid (collection: any) {
+  return collection === undefined ||
+    (typeof collection === 'string' && isActivityPubUrlValid(collection)) ||
+    (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t)))
+}
+
+export function isObjectValid (object: any) {
+  return exists(object) &&
+    (
+      isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id)
+    )
+}
+
+export function isActivityPubHTMLUrlValid (url: ActivityHtmlUrlObject) {
+  return url?.type === 'Link' &&
+    url.mediaType === 'text/html' &&
+    isActivityPubUrlValid(url.href)
+}
+
+export function setValidAttributedTo (obj: any) {
+  if (Array.isArray(obj.attributedTo) === false) {
+    obj.attributedTo = []
+    return true
+  }
+
+  obj.attributedTo = obj.attributedTo.filter(a => {
+    return isActivityPubUrlValid(a) ||
+      ((a.type === 'Group' || a.type === 'Person') && isActivityPubUrlValid(a.id))
+  })
+
+  return true
+}
+
+export function isActivityPubVideoDurationValid (value: string) {
+  // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
+  return exists(value) &&
+    typeof value === 'string' &&
+    value.startsWith('PT') &&
+    value.endsWith('S')
+}

server/core/helpers/custom-validators/activitypub/player-settings.ts

@@ -0,0 +1,13 @@
+import { PlayerSettingsObject } from '@peertube/peertube-models'
+import { isPlayerChannelThemeSettingValid, isPlayerVideoThemeSettingValid } from '../player-settings.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+export function sanitizeAndCheckPlayerSettingsObject (settings: PlayerSettingsObject, target: 'video' | 'channel') {
+  if (!settings) return false
+
+  if (settings.type !== 'PlayerSettings') return false
+  if (target === 'video' && !isPlayerVideoThemeSettingValid(settings.theme)) return false
+  if (target === 'channel' && !isPlayerChannelThemeSettingValid(settings.theme)) return false
+
+  return isActivityPubUrlValid(settings.id)
+}

server/core/helpers/custom-validators/activitypub/playlist.ts

@@ -0,0 +1,22 @@
+import { PlaylistElementObject, PlaylistObject } from '@peertube/peertube-models'
+import validator from 'validator'
+import { exists, isDateValid, isUUIDValid } from '../misc.js'
+import { isVideoPlaylistNameValid } from '../video-playlists.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+export function isPlaylistObjectValid (object: PlaylistObject) {
+  if (object?.type !== 'Playlist') return false
+
+  return validator.default.isInt(object.totalItems + '') &&
+    isVideoPlaylistNameValid(object.name) &&
+    isUUIDValid(object.uuid) &&
+    isDateValid(object.published) &&
+    isDateValid(object.updated)
+}
+
+export function isPlaylistElementObjectValid (object: PlaylistElementObject) {
+  return exists(object) &&
+    object.type === 'PlaylistElement' &&
+    validator.default.isInt(object.position + '') &&
+    isActivityPubUrlValid(object.url)
+}

server/core/helpers/custom-validators/activitypub/signature.ts

@@ -0,0 +1,22 @@
+import { exists } from '../misc.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+function isSignatureTypeValid (signatureType: string) {
+  return exists(signatureType) && signatureType === 'RsaSignature2017'
+}
+
+function isSignatureCreatorValid (signatureCreator: string) {
+  return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator)
+}
+
+function isSignatureValueValid (signatureValue: string) {
+  return exists(signatureValue) && signatureValue.length > 0
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isSignatureTypeValid,
+  isSignatureCreatorValid,
+  isSignatureValueValid
+}

server/core/helpers/custom-validators/activitypub/video-chapters.ts

@@ -0,0 +1,15 @@
+import { isArray } from '../misc.js'
+import { isVideoChapterTitleValid, isVideoChapterTimecodeValid } from '../video-chapters.js'
+import { isActivityPubUrlValid } from './misc.js'
+import { VideoChaptersObject } from '@peertube/peertube-models'
+
+export function isVideoChaptersObjectValid (object: VideoChaptersObject) {
+  if (!object) return false
+  if (!isActivityPubUrlValid(object.id)) return false
+
+  if (!isArray(object.hasPart)) return false
+
+  return object.hasPart.every(part => {
+    return isVideoChapterTitleValid(part.name) && isVideoChapterTimecodeValid(part.startOffset)
+  })
+}

server/core/helpers/custom-validators/activitypub/video-comments.ts

@@ -0,0 +1,57 @@
+import { ActivityTombstoneObject, VideoCommentObject } from '@peertube/peertube-models'
+import { hasAPPublic } from '@server/helpers/activity-pub-utils.js'
+import validator from 'validator'
+import { exists, isDateValid } from '../misc.js'
+import { isActivityPubUrlValid } from './misc.js'
+
+function sanitizeAndCheckVideoCommentObject (comment: VideoCommentObject | ActivityTombstoneObject) {
+  if (!comment) return false
+
+  if (!isCommentTypeValid(comment)) return false
+
+  normalizeComment(comment)
+
+  if (comment.type === 'Tombstone') {
+    return isActivityPubUrlValid(comment.id) &&
+      isDateValid(comment.published) &&
+      isDateValid(comment.deleted) &&
+      isActivityPubUrlValid(comment.url)
+  }
+
+  return isActivityPubUrlValid(comment.id) &&
+    isCommentContentValid(comment.content) &&
+    isActivityPubUrlValid(comment.inReplyTo) &&
+    isDateValid(comment.published) &&
+    isActivityPubUrlValid(comment.url) &&
+    (!exists(comment.replyApproval) || isActivityPubUrlValid(comment.replyApproval)) &&
+    (hasAPPublic(comment.to) || hasAPPublic(comment.cc)) // Only accept public or unlisted comments
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  sanitizeAndCheckVideoCommentObject
+}
+
+// ---------------------------------------------------------------------------
+
+function isCommentContentValid (content: any) {
+  return exists(content) && validator.default.isLength('' + content, { min: 1 })
+}
+
+function normalizeComment (comment: any) {
+  if (!comment) return
+
+  if (typeof comment.url !== 'string') {
+    if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url
+    else comment.url = comment.id
+  }
+}
+
+function isCommentTypeValid (comment: any): boolean {
+  if (comment.type === 'Note') return true
+
+  if (comment.type === 'Tombstone' && comment.formerType === 'Note') return true
+
+  return false
+}

server/core/helpers/custom-validators/activitypub/videos.ts

@@ -0,0 +1,279 @@
+import { arrayify } from '@peertube/peertube-core-utils'
+import {
+  ActivityCaptionUrlObject,
+  ActivityPubStoryboard,
+  ActivityTrackerUrlObject,
+  ActivityVideoFileMetadataUrlObject,
+  LiveVideoLatencyMode,
+  VideoCommentPolicy,
+  VideoObject,
+  VideoState
+} from '@peertube/peertube-models'
+import { logger } from '@server/helpers/logger.js'
+import { spdxToPeertubeLicence } from '@server/helpers/video.js'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
+import { peertubeTruncate } from '../../core-utils.js'
+import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js'
+import { isLiveLatencyModeValid } from '../video-lives.js'
+import {
+  isVideoCommentsPolicyValid,
+  isVideoDescriptionValid,
+  isVideoDurationValid,
+  isVideoNameValid,
+  isVideoStateValid,
+  isVideoTagValid,
+  isVideoViewsValid
+} from '../videos.js'
+import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc.js'
+
+export function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
+  return isBaseActivityValid(activity, 'Update') &&
+    sanitizeAndCheckVideoTorrentObject(activity.object)
+}
+
+export function sanitizeAndCheckVideoTorrentObject (video: VideoObject) {
+  if (video?.type !== 'Video') return false
+
+  const fail = (field: string) => {
+    logger.debug(`Video field is not valid to PeerTube: ${field}`, { video })
+    return false
+  }
+
+  if (!setValidRemoteTags(video)) return fail('tags')
+  if (!setValidRemoteVideoUrls(video)) return fail('urls')
+  if (!setRemoteVideoContent(video)) return fail('content')
+  if (!setValidAttributedTo(video)) return fail('attributedTo')
+  if (!setValidRemoteCaptions(video)) return fail('captions')
+  if (!setValidRemoteIcon(video)) return fail('icons')
+  if (!setValidStoryboard(video)) return fail('preview (storyboard)')
+  if (!setValidLicence(video)) return fail('licence')
+
+  // Default attributes
+  if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED
+  if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false
+  if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true
+  if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false
+  if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false
+  if (!isBooleanValid(video.permanentLive)) video.permanentLive = false
+  if (!isBooleanValid(video.sensitive)) video.sensitive = false
+  if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT
+
+  if (video.commentsPolicy) {
+    if (!isVideoCommentsPolicyValid(video.commentsPolicy)) {
+      video.commentsPolicy = VideoCommentPolicy.DISABLED
+    }
+  } else {
+    video.commentsPolicy = VideoCommentPolicy.DISABLED
+  }
+
+  if (!isActivityPubUrlValid(video.id)) return fail('id')
+  if (!isVideoNameValid(video.name)) return fail('name')
+
+  if (!isActivityPubVideoDurationValid(video.duration)) return fail('duration format')
+  if (!isVideoDurationValid(video.duration.replace(/[^0-9]+/g, ''))) return fail('duration')
+
+  if (!isUUIDValid(video.uuid)) return fail('uuid')
+
+  if (exists(video.category) && !isRemoteNumberIdentifierValid(video.category)) return fail('category')
+  if (exists(video.language) && !isRemoteStringIdentifierValid(video.language)) return fail('language')
+
+  if (!isVideoViewsValid(video.views)) return fail('views')
+  if (!isDateValid(video.published)) return fail('published')
+  if (!isDateValid(video.updated)) return fail('updated')
+
+  if (exists(video.originallyPublishedAt) && !isDateValid(video.originallyPublishedAt)) return fail('originallyPublishedAt')
+  if (exists(video.uploadDate) && !isDateValid(video.uploadDate)) return fail('uploadDate')
+  if (exists(video.content) && !isRemoteVideoContentValid(video.mediaType, video.content)) return fail('mediaType/content')
+
+  if (video.attributedTo.length === 0) return fail('attributedTo')
+
+  return true
+}
+
+export function isRemoteVideoUrlValid (url: any) {
+  return url.type === 'Link' &&
+      // Video file link
+      (
+        MIMETYPES.AP_VIDEO.MIMETYPE_EXT[url.mediaType] &&
+        isActivityPubUrlValid(url.href) &&
+        validator.default.isInt(url.height + '', { min: 0 }) &&
+        validator.default.isInt(url.size + '', { min: 0 }) &&
+        (!url.fps || validator.default.isInt(url.fps + '', { min: -1 }))
+      ) ||
+    // Torrent link
+    (
+      MIMETYPES.AP_TORRENT.MIMETYPE_EXT[url.mediaType] &&
+      isActivityPubUrlValid(url.href) &&
+      validator.default.isInt(url.height + '', { min: 0 })
+    ) ||
+    // Magnet link
+    (
+      MIMETYPES.AP_MAGNET.MIMETYPE_EXT[url.mediaType] &&
+      validator.default.isLength(url.href, { min: 5 }) &&
+      validator.default.isInt(url.height + '', { min: 0 })
+    ) ||
+    // HLS playlist link
+    (
+      (url.mediaType || url.mimeType) === 'application/x-mpegURL' &&
+      isActivityPubUrlValid(url.href) &&
+      isArray(url.tag)
+    ) ||
+    isAPVideoTrackerUrlObject(url) ||
+    isAPVideoFileUrlMetadataObject(url)
+}
+
+export function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject {
+  return url?.type === 'Link' &&
+    url.mediaType === 'application/json' &&
+    isArray(url.rel) && url.rel.includes('metadata')
+}
+
+export function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject {
+  return isArray(url.rel) &&
+    url.rel.includes('tracker') &&
+    isActivityPubUrlValid(url.href)
+}
+
+export function setAPCaptionUrlObject (url: any): url is ActivityCaptionUrlObject {
+  if (url?.type !== 'Link') return false
+  if (!isActivityPubUrlValid(url.href)) return false
+
+  if (!url.mediaType && url.href.endsWith('.vtt')) {
+    url.mediaType = 'text/vtt'
+  }
+
+  return url.mediaType === 'text/vtt' || url.mediaType === 'application/x-mpegURL'
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function setValidRemoteTags (video: VideoObject) {
+  if (Array.isArray(video.tag) === false) video.tag = []
+
+  video.tag = video.tag.filter(t => {
+    return (t.type === 'Hashtag' && isVideoTagValid(t.name)) ||
+      (t.type === 'SensitiveTag' && !!t.name)
+  })
+
+  return true
+}
+
+function setValidRemoteCaptions (video: VideoObject) {
+  if (!video.subtitleLanguage) video.subtitleLanguage = []
+
+  if (Array.isArray(video.subtitleLanguage) === false) return false
+
+  video.subtitleLanguage = video.subtitleLanguage.filter(caption => {
+    if (typeof caption.url === 'string') {
+      if (isActivityPubUrlValid(caption.url)) {
+        caption.url = [
+          {
+            type: 'Link',
+            href: caption.url,
+            mediaType: 'text/vtt'
+          }
+        ]
+      } else {
+        caption.url = []
+      }
+    } else {
+      caption.url = arrayify(caption.url).filter(u => setAPCaptionUrlObject(u))
+    }
+
+    return caption.url.length > 0 && isRemoteStringIdentifierValid(caption)
+  })
+
+  return true
+}
+
+function isRemoteNumberIdentifierValid (data: any) {
+  return validator.default.isInt(data.identifier, { min: 0 })
+}
+
+function isRemoteStringIdentifierValid (data: any) {
+  return typeof data.identifier === 'string'
+}
+
+function isRemoteVideoContentValid (mediaType: string, content: string) {
+  return (mediaType === 'text/markdown' || mediaType === 'text/html') && isVideoDescriptionValid(content)
+}
+
+function setValidRemoteIcon (video: any) {
+  if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ]
+  if (!video.icon) video.icon = []
+
+  video.icon = video.icon.filter(icon => {
+    return icon.type === 'Image' &&
+      isActivityPubUrlValid(icon.url) &&
+      icon.mediaType === 'image/jpeg' &&
+      validator.default.isInt(icon.width + '', { min: 0 }) &&
+      validator.default.isInt(icon.height + '', { min: 0 })
+  })
+
+  return video.icon.length !== 0
+}
+
+function setValidRemoteVideoUrls (video: any) {
+  if (Array.isArray(video.url) === false) return false
+
+  video.url = video.url.filter(u => isRemoteVideoUrlValid(u))
+
+  return true
+}
+
+function setRemoteVideoContent (video: VideoObject) {
+  if (video.content) {
+    video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max })
+  }
+
+  return true
+}
+
+function setValidLicence (video: VideoObject) {
+  if (!exists(video.licence)) return true
+
+  if (validator.default.isInt(video.licence.identifier)) return isRemoteNumberIdentifierValid(video.licence)
+
+  const spdx = spdxToPeertubeLicence(video.licence.identifier)
+  video.licence.identifier = spdx
+    ? spdx + ''
+    : undefined
+
+  return true
+}
+
+function setValidStoryboard (video: VideoObject) {
+  if (!video.preview) return true
+  if (!Array.isArray(video.preview)) return false
+
+  video.preview = video.preview.filter(p => isStorybordValid(p))
+
+  return true
+}
+
+function isStorybordValid (preview: ActivityPubStoryboard) {
+  if (!preview) return false
+
+  if (
+    preview.type !== 'Image' ||
+    !isArray(preview.rel) ||
+    !preview.rel.includes('storyboard')
+  ) {
+    return false
+  }
+
+  preview.url = preview.url.filter(u => {
+    return u.mediaType === 'image/jpeg' &&
+      isActivityPubUrlValid(u.href) &&
+      validator.default.isInt(u.width + '', { min: 0 }) &&
+      validator.default.isInt(u.height + '', { min: 0 }) &&
+      validator.default.isInt(u.tileWidth + '', { min: 0 }) &&
+      validator.default.isInt(u.tileHeight + '', { min: 0 }) &&
+      isActivityPubVideoDurationValid(u.tileDuration)
+  })
+
+  return preview.url.length !== 0
+}

server/core/helpers/custom-validators/activitypub/watch-action.ts

@@ -0,0 +1,41 @@
+import { WatchActionObject } from '@peertube/peertube-models'
+import { isDateValid, isUUIDValid } from '../misc.js'
+import { isVideoTimeValid } from '../video-view.js'
+import { isActivityPubVideoDurationValid, isObjectValid } from './misc.js'
+
+function isWatchActionObjectValid (action: WatchActionObject) {
+  if (action?.type !== 'WatchAction') return false
+
+  return isObjectValid(action.id) &&
+    isActivityPubVideoDurationValid(action.duration) &&
+    isDateValid(action.startTime) &&
+    isDateValid(action.endTime) &&
+    isLocationValid(action.location) &&
+    isUUIDValid(action.uuid) &&
+    isObjectValid(action.object) &&
+    areWatchSectionsValid(action.watchSections)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isWatchActionObjectValid
+}
+
+// ---------------------------------------------------------------------------
+
+function isLocationValid (location: any) {
+  if (!location) return true
+  if (typeof location !== 'object') return false
+
+  if (location.addressCountry && typeof location.addressCountry !== 'string') return false
+  if (location.addressRegion && typeof location.addressRegion !== 'string') return false
+
+  return true
+}
+
+function areWatchSectionsValid (sections: WatchActionObject['watchSections']) {
+  return Array.isArray(sections) && sections.every(s => {
+    return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp)
+  })
+}

server/core/helpers/custom-validators/actor-images.ts

@@ -0,0 +1,17 @@
+import { UploadFilesForCheck } from 'express'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { isFileValid } from './misc.js'
+
+const imageMimeTypes = CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME
+  .map(v => v.replace('.', ''))
+  .join('|')
+const imageMimeTypesRegex = `image/(${imageMimeTypes})`
+
+export function isActorImageFile (files: UploadFilesForCheck, fieldname: string) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: imageMimeTypesRegex,
+    field: fieldname,
+    maxSize: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max
+  })
+}

server/core/helpers/custom-validators/browse-videos.ts

@@ -0,0 +1,53 @@
+import { t } from '../i18n.js'
+
+export function getBrowseVideosDefaultSortError (value: string, enabledTrendingAlgorithms: string[], language?: string) {
+  const availableOptions = [ '-publishedAt', '-originallyPublishedAt', 'name', '-trending', '-hot', '-likes', '-views' ]
+
+  if (availableOptions.includes(value) === false) {
+    const options = availableOptions.join(' or ')
+
+    // Only use `t` if language is provided, because i18n might not be initialized yet and can return `undefined`
+    if (language) {
+      return t('Browse videos default sort should be {options}, instead of {value}', language, { options, value })
+    }
+
+    return `Browse videos default sort should be ${options}, instead of ${value}`
+  }
+
+  const trendingSortAlgorithmMap = new Map<string, string>([
+    [ '-trending', 'most-viewed' ],
+    [ '-hot', 'hot' ],
+    [ '-likes', 'most-liked' ]
+  ])
+  const currentTrendingSortAlgorithm = trendingSortAlgorithmMap.get(value)
+
+  if (currentTrendingSortAlgorithm && enabledTrendingAlgorithms.includes(currentTrendingSortAlgorithm) === false) {
+    if (language) {
+      return t(
+        'Trending videos algorithm {currentTrendingSortAlgorithm} should be enabled if browse videos default sort is {value}',
+        language,
+        { currentTrendingSortAlgorithm, value }
+      )
+    }
+
+    return `Trending videos algorithm ${currentTrendingSortAlgorithm} should be enabled if browse videos default sort is ${value}`
+  }
+
+  return null
+}
+
+export function getBrowseVideosDefaultScopeError (value: string, language?: string) {
+  const availableOptions = [ 'local', 'federated' ]
+
+  if (availableOptions.includes(value) === false) {
+    const options = availableOptions.join(' or ')
+
+    if (language) {
+      return t('Browse videos default scope should be {options}, instead of {value}', language, { options, value })
+    }
+
+    return `Browse videos default scope should be ${options}, instead of ${value}`
+  }
+
+  return null
+}

server/core/helpers/custom-validators/bulk.ts

@@ -0,0 +1,5 @@
+import { BulkRemoveCommentsOfBody } from '@peertube/peertube-models'
+
+export function isBulkRemoveCommentsOfScopeValid (value: BulkRemoveCommentsOfBody['scope']) {
+  return value === 'my-videos' || value === 'instance' || value === 'my-videos-and-collaborations'
+}

server/core/helpers/custom-validators/config.ts

@@ -0,0 +1,23 @@
+import { LogoType } from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS, MIMETYPES } from '@server/initializers/constants.js'
+import { UploadFilesForCheck } from 'express'
+import { isFileValid } from './misc.js'
+
+const logoTypes = new Set<LogoType>([ 'favicon', 'header-square', 'header-wide', 'opengraph' ])
+
+export function isConfigLogoTypeValid (value: LogoType) {
+  return logoTypes.has(value)
+}
+
+const imageMimeTypesRegex = Object.keys(MIMETYPES.LOGO_IMAGE.MIMETYPE_EXT)
+  .map(mimeType => `(${mimeType.replace('+', '\\+')})`)
+  .join('|')
+
+export function isLogoImageFile (files: UploadFilesForCheck, fieldname: string) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: imageMimeTypesRegex,
+    field: fieldname,
+    maxSize: CONSTRAINTS_FIELDS.LOGO.IMAGE.FILE_SIZE.max
+  })
+}

server/core/helpers/custom-validators/feeds.ts

@@ -0,0 +1,23 @@
+import { exists } from './misc.js'
+
+function isValidRSSFeed (value: string) {
+  if (!exists(value)) return false
+
+  const feedExtensions = [
+    'xml',
+    'json',
+    'json1',
+    'rss',
+    'rss2',
+    'atom',
+    'atom1'
+  ]
+
+  return feedExtensions.includes(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidRSSFeed
+}

server/core/helpers/custom-validators/follows.ts

@@ -0,0 +1,30 @@
+import { exists, isArray } from './misc.js'
+import { FollowState } from '@peertube/peertube-models'
+
+function isFollowStateValid (value: FollowState) {
+  if (!exists(value)) return false
+
+  return value === 'pending' || value === 'accepted' || value === 'rejected'
+}
+
+function isRemoteHandleValid (value: string) {
+  if (!exists(value)) return false
+  if (typeof value !== 'string') return false
+
+  return value.includes('@')
+}
+
+function isEachUniqueHandleValid (handles: string[]) {
+  return isArray(handles) &&
+    handles.every(handle => {
+      return isRemoteHandleValid(handle) && handles.indexOf(handle) === handles.lastIndexOf(handle)
+    })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isFollowStateValid,
+  isRemoteHandleValid,
+  isEachUniqueHandleValid
+}

server/core/helpers/custom-validators/jobs.ts

@@ -0,0 +1,21 @@
+import { JobState } from '@peertube/peertube-models'
+import { jobTypes } from '@server/lib/job-queue/job-queue.js'
+import { exists } from './misc.js'
+
+const jobStates = new Set<JobState>([ 'active', 'completed', 'failed', 'waiting', 'delayed', 'paused', 'waiting-children', 'prioritized' ])
+
+function isValidJobState (value: JobState) {
+  return exists(value) && jobStates.has(value)
+}
+
+function isValidJobType (value: any) {
+  return exists(value) && jobTypes.includes(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  jobStates,
+  isValidJobState,
+  isValidJobType
+}

server/core/helpers/custom-validators/logs.ts

@@ -0,0 +1,42 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import { ClientLogLevel, ServerLogLevel } from '@peertube/peertube-models'
+import { exists } from './misc.js'
+
+const serverLogLevels = new Set<ServerLogLevel>([ 'debug', 'info', 'warn', 'error' ])
+const clientLogLevels = new Set<ClientLogLevel>([ 'warn', 'error' ])
+
+function isValidLogLevel (value: any) {
+  return exists(value) && serverLogLevels.has(value)
+}
+
+function isValidClientLogMessage (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE)
+}
+
+function isValidClientLogLevel (value: any) {
+  return exists(value) && clientLogLevels.has(value)
+}
+
+function isValidClientLogStackTrace (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE)
+}
+
+function isValidClientLogMeta (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META)
+}
+
+function isValidClientLogUserAgent (value: any) {
+  return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidLogLevel,
+  isValidClientLogMessage,
+  isValidClientLogStackTrace,
+  isValidClientLogMeta,
+  isValidClientLogLevel,
+  isValidClientLogUserAgent
+}

server/core/helpers/custom-validators/metrics.ts

@@ -0,0 +1,3 @@
+export function isValidPlayerMode (value: any) {
+  return value === 'web-video' || value === 'p2p-media-loader'
+}

server/core/helpers/custom-validators/misc.ts

@@ -0,0 +1,196 @@
+import { isShortUUID, shortToUUID } from '@peertube/peertube-node-utils'
+import { UploadFilesForCheck } from 'express'
+import 'multer'
+import { sep } from 'path'
+import validator from 'validator'
+
+export function exists (value: any) {
+  return value !== undefined && value !== null
+}
+
+export function isSafePath (p: string) {
+  return exists(p) &&
+    (p + '').split(sep).every(part => {
+      return [ '..' ].includes(part) === false
+    })
+}
+
+export function isSafeFilename (filename: string, extension?: string) {
+  const regex = extension
+    ? new RegExp(`^[a-z0-9-]+\\.${extension}$`)
+    : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`)
+
+  return typeof filename === 'string' && !!filename.match(regex)
+}
+
+export function isSafePeerTubeFilenameWithoutExtension (filename: string) {
+  return filename.match(/^[a-z0-9-]+$/)
+}
+
+// ---------------------------------------------------------------------------
+
+export function isArray (value: any): value is any[] {
+  return Array.isArray(value)
+}
+
+export function isNotEmptyIntArray (value: any) {
+  return Array.isArray(value) && value.every(v => validator.default.isInt('' + v)) && value.length !== 0
+}
+
+export function isNotEmptyStringArray (value: any) {
+  return Array.isArray(value) && value.every(v => typeof v === 'string' && v.length !== 0) && value.length !== 0
+}
+
+export function hasArrayLength (value: unknown[], options: { min?: number, max?: number }) {
+  if (options.min !== undefined && value.length < options.min) return false
+  if (options.max !== undefined && value.length > options.max) return false
+
+  return true
+}
+
+export function isArrayOf (value: any, validator: (value: any) => boolean) {
+  return isArray(value) && value.every(v => validator(v))
+}
+
+// ---------------------------------------------------------------------------
+
+export function isDateValid (value: string) {
+  return exists(value) && validator.default.isISO8601(value)
+}
+
+export function isIdValid (value: string) {
+  return exists(value) && validator.default.isInt('' + value)
+}
+
+export function isUUIDValid (value: string) {
+  return exists(value) && validator.default.isUUID('' + value, 4)
+}
+
+export function areUUIDsValid (values: string[]) {
+  return isArray(values) && values.every(v => isUUIDValid(v))
+}
+
+export function isIdOrUUIDValid (value: string) {
+  return isIdValid(value) || isUUIDValid(value)
+}
+
+export function isBooleanValid (value: any) {
+  return typeof value === 'boolean' || (typeof value === 'string' && validator.default.isBoolean(value))
+}
+
+export function isIntOrNull (value: any) {
+  return value === null || validator.default.isInt('' + value)
+}
+
+// ---------------------------------------------------------------------------
+
+export function isFileValid (options: {
+  files: UploadFilesForCheck
+
+  maxSize: number | null
+  mimeTypeRegex: string | null
+
+  field?: string
+
+  optional?: boolean // Default false
+}) {
+  const { files, mimeTypeRegex, field, maxSize, optional = false } = options
+
+  // Should have files
+  if (!files) return optional
+
+  const fileArray = isArray(files)
+    ? files
+    : files[field]
+
+  if (!fileArray || !isArray(fileArray) || fileArray.length === 0) {
+    return optional
+  }
+
+  // The file exists
+  const file = fileArray[0]
+  if (!file?.originalname) return false
+
+  // Check size
+  if ((maxSize !== null) && file.size > maxSize) return false
+
+  if (mimeTypeRegex === null) return true
+
+  return checkMimetypeRegex(file.mimetype, mimeTypeRegex)
+}
+
+export function checkMimetypeRegex (fileMimeType: string, mimeTypeRegex: string) {
+  return new RegExp(`^${mimeTypeRegex}$`, 'i').test(fileMimeType)
+}
+
+// ---------------------------------------------------------------------------
+
+export function toCompleteUUID (value: string) {
+  if (isShortUUID(value)) {
+    try {
+      return shortToUUID(value)
+    } catch {
+      return ''
+    }
+  }
+
+  return value
+}
+
+export function toCompleteUUIDs (values: string[]) {
+  return values.map(v => toCompleteUUID(v))
+}
+
+export function toIntOrNull (value: string) {
+  const v = toValueOrNull(value)
+
+  if (v === null || v === undefined) return v
+  if (typeof v === 'number') return v
+
+  return validator.default.toInt('' + v)
+}
+
+export function toBooleanOrNull (value: any) {
+  const v = toValueOrNull(value)
+
+  if (v === null || v === undefined) return v
+  if (typeof v === 'boolean') return v
+
+  return validator.default.toBoolean('' + v)
+}
+
+export function toValueOrNull (value: string) {
+  if (value === 'null') return null
+
+  return value
+}
+
+export function toIntArray (value: any) {
+  if (!value) return []
+  if (isArray(value) === false) return [ validator.default.toInt(value) ]
+
+  return value.map(v => validator.default.toInt(v))
+}
+
+// ---------------------------------------------------------------------------
+
+export function isStableVersionValid (value: string) {
+  if (!exists(value)) return false
+
+  const parts = (value + '').split('.')
+
+  return parts.length === 3 && parts.every(p => validator.default.isInt(p))
+}
+
+export function isStableOrUnstableVersionValid (value: string) {
+  if (!exists(value)) return false
+
+  // suffix is beta.x or alpha.x
+  const [ stable, suffix ] = value.split('-')
+  if (!isStableVersionValid(stable)) return false
+
+  const suffixRegex = /^(rc|alpha|beta)\.\d+$/
+  if (suffix && !suffixRegex.test(suffix)) return false
+
+  return true
+}

server/core/helpers/custom-validators/player-settings.ts

@@ -0,0 +1,16 @@
+import { PlayerTheme, PlayerThemeVideoSetting } from '@peertube/peertube-models'
+import { DEFAULT_CHANNEL_PLAYER_SETTING_VALUE, DEFAULT_INSTANCE_PLAYER_SETTING_VALUE } from '@server/initializers/constants.js'
+
+export function isPlayerVideoThemeSettingValid (name: PlayerThemeVideoSetting) {
+  return isPlayerChannelThemeSettingValid(name) || name === DEFAULT_CHANNEL_PLAYER_SETTING_VALUE
+}
+
+export function isPlayerChannelThemeSettingValid (name: PlayerThemeVideoSetting) {
+  return name === DEFAULT_INSTANCE_PLAYER_SETTING_VALUE || isPlayerThemeValid(name as PlayerTheme)
+}
+
+const availableThemes = new Set<PlayerTheme>([ 'galaxy', 'lucide' ])
+
+export function isPlayerThemeValid (name: PlayerTheme) {
+  return availableThemes.has(name)
+}

server/core/helpers/custom-validators/plugins.ts

@@ -0,0 +1,144 @@
+import { PluginPackageJSON, PluginType, PluginType_Type } from '@peertube/peertube-models'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { isUrlValid } from './activitypub/misc.js'
+import { exists, isArray, isSafePath } from './misc.js'
+
+const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS
+
+export function isPluginTypeValid (value: any) {
+  return exists(value) &&
+    (value === PluginType.PLUGIN || value === PluginType.THEME)
+}
+
+export function isPluginNameValid (value: string) {
+  return exists(value) &&
+    validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
+    validator.default.matches(value, /^[a-z-0-9]+$/)
+}
+
+export function isNpmPluginNameValid (value: string) {
+  return exists(value) &&
+    validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) &&
+    validator.default.matches(value, /^[a-z\-._0-9]+$/) &&
+    (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-'))
+}
+
+export function isPluginDescriptionValid (value: string) {
+  return exists(value) && validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+export function isPluginEngineValid (engine: any) {
+  return exists(engine) && exists(engine.peertube)
+}
+
+export function isPluginHomepage (value: string) {
+  return exists(value) && (!value || isUrlValid(value))
+}
+
+export function isPluginBugs (value: string) {
+  return exists(value) && (!value || isUrlValid(value))
+}
+
+export function areStaticDirectoriesValid (staticDirs: any) {
+  if (!exists(staticDirs) || typeof staticDirs !== 'object') return false
+
+  for (const key of Object.keys(staticDirs)) {
+    if (!isSafePath(staticDirs[key])) return false
+  }
+
+  return true
+}
+
+export function areClientScriptsValid (clientScripts: any[]) {
+  return isArray(clientScripts) &&
+    clientScripts.every(c => {
+      return isSafePath(c.script) && isArray(c.scopes)
+    })
+}
+
+export function areTranslationPathsValid (translations: any) {
+  if (!exists(translations) || typeof translations !== 'object') return false
+
+  for (const key of Object.keys(translations)) {
+    if (!isSafePath(translations[key])) return false
+  }
+
+  return true
+}
+
+export function areCSSPathsValid (css: any[]) {
+  return isArray(css) && css.every(c => isSafePath(c))
+}
+
+export function isThemeNameValid (name: string) {
+  return name && typeof name === 'string' &&
+    (isPluginNameValid(name) || name.startsWith('peertube-core-'))
+}
+
+export function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType_Type) {
+  let result = true
+  const badFields: string[] = []
+
+  if (!isNpmPluginNameValid(packageJSON.name)) {
+    result = false
+    badFields.push('name')
+  }
+
+  if (!isPluginDescriptionValid(packageJSON.description)) {
+    result = false
+    badFields.push('description')
+  }
+
+  if (!isPluginEngineValid(packageJSON.engine)) {
+    result = false
+    badFields.push('engine')
+  }
+
+  if (!isPluginHomepage(packageJSON.homepage)) {
+    result = false
+    badFields.push('homepage')
+  }
+
+  if (!exists(packageJSON.author)) {
+    result = false
+    badFields.push('author')
+  }
+
+  if (!isPluginBugs(packageJSON.bugs)) {
+    result = false
+    badFields.push('bugs')
+  }
+
+  if (pluginType === PluginType.PLUGIN && !isSafePath(packageJSON.library)) {
+    result = false
+    badFields.push('library')
+  }
+
+  if (!areStaticDirectoriesValid(packageJSON.staticDirs)) {
+    result = false
+    badFields.push('staticDirs')
+  }
+
+  if (!areCSSPathsValid(packageJSON.css)) {
+    result = false
+    badFields.push('css')
+  }
+
+  if (!areClientScriptsValid(packageJSON.clientScripts)) {
+    result = false
+    badFields.push('clientScripts')
+  }
+
+  if (!areTranslationPathsValid(packageJSON.translations)) {
+    result = false
+    badFields.push('translations')
+  }
+
+  return { result, badFields }
+}
+
+export function isLibraryCodeValid (library: any) {
+  return typeof library.register === 'function' &&
+    typeof library.unregister === 'function'
+}

server/core/helpers/custom-validators/runners/jobs.ts

@@ -0,0 +1,233 @@
+import {
+  LiveRTMPHLSTranscodingSuccess,
+  RunnerJobSuccessPayload,
+  RunnerJobType,
+  RunnerJobUpdatePayload,
+  TranscriptionSuccess,
+  VODAudioMergeTranscodingSuccess,
+  VODHLSTranscodingSuccess,
+  VODWebVideoTranscodingSuccess,
+  VideoStudioTranscodingSuccess,
+  GenerateStoryboardSuccess
+} from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'
+import { UploadFilesForCheck } from 'express'
+import validator from 'validator'
+import { exists, isArray, isFileValid, isSafeFilename } from '../misc.js'
+
+const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
+
+const runnerJobTypes = new Set([
+  'vod-hls-transcoding',
+  'vod-web-video-transcoding',
+  'vod-audio-merge-transcoding',
+  'live-rtmp-hls-transcoding',
+  'video-studio-transcoding',
+  'video-transcription',
+  'generate-video-storyboard'
+])
+export function isRunnerJobTypeValid (value: RunnerJobType) {
+  return runnerJobTypes.has(value)
+}
+
+export function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) {
+  return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) ||
+    isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
+    isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
+    isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) ||
+    isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) ||
+    isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files) ||
+    isRunnerJobGenerateStoryboardResultPayloadValid(value as GenerateStoryboardSuccess, type, files)
+}
+
+// ---------------------------------------------------------------------------
+
+export function isRunnerJobProgressValid (value: string) {
+  return validator.default.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS)
+}
+
+export function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) {
+  return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVODHLSUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) ||
+    isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
+    isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) ||
+    isRunnerJobTranscriptionUpdatePayloadValid(value, type, files) ||
+    isRunnerJobGenerateStoryboardUpdatePayloadValid(value, type, files)
+}
+
+// ---------------------------------------------------------------------------
+
+export function isRunnerJobTokenValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+export function isRunnerJobAbortReasonValid (value: string) {
+  return validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON)
+}
+
+export function isRunnerJobErrorMessageValid (value: string) {
+  return validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE)
+}
+
+export function isRunnerJobStateValid (value: any) {
+  return exists(value) && RUNNER_JOB_STATES[value] !== undefined
+}
+
+export function isRunnerJobArrayOfStateValid (value: any) {
+  return isArray(value) && value.every(v => isRunnerJobStateValid(v))
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function isRunnerJobVODWebVideoResultPayloadValid (
+  _value: VODWebVideoTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-web-video-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobVODHLSResultPayloadValid (
+  _value: VODHLSTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-hls-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) &&
+    isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobVODAudioMergeResultPayloadValid (
+  _value: VODAudioMergeTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'vod-audio-merge-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobLiveRTMPHLSResultPayloadValid (
+  value: LiveRTMPHLSTranscodingSuccess,
+  type: RunnerJobType
+) {
+  return type === 'live-rtmp-hls-transcoding' && isEmptyUpdatePayload(value)
+}
+
+function isRunnerJobVideoStudioResultPayloadValid (
+  _value: VideoStudioTranscodingSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'video-studio-transcoding' &&
+    isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobTranscriptionResultPayloadValid (
+  value: TranscriptionSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'video-transcription' &&
+    isFileValid({ files, field: 'payload[vttFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+function isRunnerJobGenerateStoryboardResultPayloadValid (
+  value: GenerateStoryboardSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'generate-video-storyboard' &&
+    isFileValid({ files, field: 'payload[storyboardFile]', mimeTypeRegex: null, maxSize: null })
+}
+
+// ---------------------------------------------------------------------------
+
+function isRunnerJobVODWebVideoUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-web-video-transcoding' && isEmptyUpdatePayload(value)
+}
+
+function isRunnerJobVODHLSUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-hls-transcoding' && isEmptyUpdatePayload(value)
+}
+
+function isRunnerJobVODAudioMergeUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'vod-audio-merge-transcoding' && isEmptyUpdatePayload(value)
+}
+
+function isRunnerJobTranscriptionUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'video-transcription' && isEmptyUpdatePayload(value)
+}
+
+function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files
+
+  result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true })
+
+  result &&= isFileValid({
+    files,
+    field: 'payload[resolutionPlaylistFile]',
+    mimeTypeRegex: null,
+    maxSize: null,
+    optional: !value.resolutionPlaylistFilename
+  })
+
+  if (files['payload[resolutionPlaylistFile]']) {
+    result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8')
+  }
+
+  return result &&
+    isSafeFilename(value.videoChunkFilename, 'ts') &&
+    (
+      (
+        value.type === 'remove-chunk'
+      ) ||
+      (
+        value.type === 'add-chunk' &&
+        isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null })
+      )
+    )
+}
+
+function isRunnerJobVideoStudioUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'video-studio-transcoding' && isEmptyUpdatePayload(value)
+}
+
+function isRunnerJobGenerateStoryboardUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'generate-video-storyboard' && isEmptyUpdatePayload(value)
+}
+
+function isEmptyUpdatePayload (value: any): boolean {
+  return !value || (typeof value === 'object' && Object.keys(value).length === 0)
+}

server/core/helpers/custom-validators/runners/runners.ts

@@ -0,0 +1,30 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import { exists } from '../misc.js'
+
+const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS
+
+function isRunnerRegistrationTokenValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerTokenValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN)
+}
+
+function isRunnerNameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME)
+}
+
+function isRunnerDescriptionValid (value: string) {
+  return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isRunnerRegistrationTokenValid,
+  isRunnerTokenValid,
+  isRunnerNameValid,
+  isRunnerDescriptionValid
+}

server/core/helpers/custom-validators/search.ts

@@ -0,0 +1,37 @@
+import validator from 'validator'
+import { SearchTargetType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { exists, isArray } from './misc.js'
+
+function isNumberArray (value: any) {
+  return isArray(value) && value.every(v => validator.default.isInt('' + v))
+}
+
+function isStringArray (value: any) {
+  return isArray(value) && value.every(v => typeof v === 'string')
+}
+
+function isBooleanBothQueryValid (value: any) {
+  return value === 'true' || value === 'false' || value === 'both'
+}
+
+function isSearchTargetValid (value: SearchTargetType) {
+  if (!exists(value)) return true
+
+  const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX
+
+  if (value === 'local') return true
+
+  if (value === 'search-index' && searchIndexConfig.ENABLED) return true
+
+  return false
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isNumberArray,
+  isStringArray,
+  isBooleanBothQueryValid,
+  isSearchTargetValid
+}

server/core/helpers/custom-validators/servers.ts

@@ -0,0 +1,42 @@
+import validator from 'validator'
+import { CONFIG } from '@server/initializers/config.js'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists, isArray } from './misc.js'
+
+function isHostValid (host: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true
+  }
+
+  // We validate 'localhost', so we don't have the top level domain
+  if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') {
+    isURLOptions.require_tld = false
+  }
+
+  return exists(host) && validator.default.isURL(host, isURLOptions) && host.split('://').length === 1
+}
+
+function isEachUniqueHostValid (hosts: string[]) {
+  return isArray(hosts) &&
+    hosts.every(host => {
+      return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host)
+    })
+}
+
+function isValidContactBody (value: any) {
+  return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.BODY)
+}
+
+function isValidContactFromName (value: any) {
+  return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.FROM_NAME)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidContactBody,
+  isValidContactFromName,
+  isEachUniqueHostValid,
+  isHostValid
+}

server/core/helpers/custom-validators/user-notifications.ts

@@ -0,0 +1,23 @@
+import validator from 'validator'
+import { UserNotificationSettingValue } from '@peertube/peertube-models'
+import { exists } from './misc.js'
+
+function isUserNotificationTypeValid (value: any) {
+  return exists(value) && validator.default.isInt('' + value)
+}
+
+function isUserNotificationSettingValid (value: any) {
+  return exists(value) &&
+    validator.default.isInt('' + value) &&
+    (
+      value === UserNotificationSettingValue.NONE ||
+      value === UserNotificationSettingValue.WEB ||
+      value === UserNotificationSettingValue.EMAIL ||
+      value === (UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL)
+    )
+}
+
+export {
+  isUserNotificationSettingValid,
+  isUserNotificationTypeValid
+}

server/core/helpers/custom-validators/user-registration.ts

@@ -0,0 +1,25 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, USER_REGISTRATION_STATES } from '../../initializers/constants.js'
+import { exists } from './misc.js'
+
+const USER_REGISTRATIONS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USER_REGISTRATIONS
+
+function isRegistrationStateValid (value: string) {
+  return exists(value) && USER_REGISTRATION_STATES[value] !== undefined
+}
+
+function isRegistrationModerationResponseValid (value: string) {
+  return exists(value) && validator.default.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.MODERATOR_MESSAGE)
+}
+
+function isRegistrationReasonValid (value: string) {
+  return exists(value) && validator.default.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.REASON_MESSAGE)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isRegistrationStateValid,
+  isRegistrationModerationResponseValid,
+  isRegistrationReasonValid
+}

server/core/helpers/custom-validators/users.ts

@@ -0,0 +1,114 @@
+import { AVAILABLE_LOCALES } from '@peertube/peertube-core-utils'
+import { UserRole } from '@peertube/peertube-models'
+import validator from 'validator'
+import { isEmailEnabled } from '../../initializers/config.js'
+import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants.js'
+import { exists, isArray, isBooleanValid } from './misc.js'
+
+const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS
+
+export function isUserPasswordValid (value: string) {
+  return validator.default.isLength(value, { min: USERS_CONSTRAINTS_FIELDS.PASSWORD.min }) &&
+    isUserPasswordTooLong(value) !== true
+}
+
+export function isUserPasswordTooLong (value: string) {
+  return !validator.default.isLength(value, { max: USERS_CONSTRAINTS_FIELDS.PASSWORD.max }) ||
+    Buffer.byteLength(value, 'utf8') > USERS_CONSTRAINTS_FIELDS.PASSWORD.maxBytes
+}
+
+export function isUserPasswordValidOrEmpty (value: string) {
+  // Empty password is only possible if emailing is enabled.
+  if (value === '') return isEmailEnabled()
+
+  return isUserPasswordValid(value)
+}
+
+export function isUserVideoQuotaValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA)
+}
+
+export function isUserVideoQuotaDailyValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY)
+}
+
+export function isUserUsernameValid (value: string) {
+  return exists(value) &&
+    validator.default.matches(value, new RegExp(`^[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?$`)) &&
+    validator.default.isLength(value, USERS_CONSTRAINTS_FIELDS.USERNAME)
+}
+
+export function isUserDisplayNameValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.NAME))
+}
+
+export function isUserDescriptionValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.DESCRIPTION))
+}
+
+export function isUserEmailVerifiedValid (value: any) {
+  return isBooleanValid(value)
+}
+
+const nsfwPolicies = new Set(Object.values(NSFW_POLICY_TYPES))
+export function isUserNSFWPolicyValid (value: any) {
+  return exists(value) && nsfwPolicies.has(value)
+}
+
+export function isUserP2PEnabledValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserVideosHistoryEnabledValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserAutoPlayVideoValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserVideoLanguages (value: any) {
+  return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max)
+}
+
+export function isUserLanguage (value: any) {
+  return value === null || AVAILABLE_LOCALES.includes(value)
+}
+
+export function isUserAdminFlagsValid (value: any) {
+  return exists(value) && validator.default.isInt('' + value)
+}
+
+export function isUserBlockedValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserAutoPlayNextVideoValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserAutoPlayNextVideoPlaylistValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserEmailPublicValid (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserNoModal (value: any) {
+  return isBooleanValid(value)
+}
+
+export function isUserBlockedReasonValid (value: any) {
+  return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.BLOCKED_REASON))
+}
+
+export function isUserRoleValid (value: any) {
+  return exists(value) &&
+    validator.default.isInt('' + value) &&
+    [ UserRole.ADMINISTRATOR, UserRole.MODERATOR, UserRole.USER ].includes(value)
+}
+
+export function isUserFeatureInfo (value: string) {
+  return exists(value) && validator.default.isInt(value + '')
+}

server/core/helpers/custom-validators/video-blacklist.ts

@@ -0,0 +1,22 @@
+import validator from 'validator'
+import { VideoBlacklistType } from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists } from './misc.js'
+
+const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST
+
+function isVideoBlacklistReasonValid (value: string) {
+  return value === null || validator.default.isLength(value, VIDEO_BLACKLIST_CONSTRAINTS_FIELDS.REASON)
+}
+
+function isVideoBlacklistTypeValid (value: any) {
+  return exists(value) &&
+    (value === VideoBlacklistType.AUTO_BEFORE_PUBLISHED || value === VideoBlacklistType.MANUAL)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoBlacklistReasonValid,
+  isVideoBlacklistTypeValid
+}

server/core/helpers/custom-validators/video-captions.ts

@@ -0,0 +1,43 @@
+import { UploadFilesForCheck } from 'express'
+import { readFile } from 'fs/promises'
+import { getFileSize } from '@peertube/peertube-node-utils'
+import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants.js'
+import { logger } from '../logger.js'
+import { exists, isFileValid } from './misc.js'
+
+function isVideoCaptionLanguageValid (value: any) {
+  return exists(value) && VIDEO_LANGUAGES[value] !== undefined
+}
+
+// MacOS sends application/octet-stream
+const videoCaptionTypesRegex = [ ...Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT), 'application/octet-stream' ]
+  .map(m => `(${m})`)
+  .join('|')
+
+function isVideoCaptionFile (files: UploadFilesForCheck, field: string) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: videoCaptionTypesRegex,
+    field,
+    maxSize: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max
+  })
+}
+
+async function isVTTFileValid (filePath: string) {
+  const size = await getFileSize(filePath)
+  const content = await readFile(filePath, 'utf8')
+
+  logger.debug('Checking VTT file %s', filePath, { size, content })
+
+  if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false
+
+  return content?.startsWith('WEBVTT')
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoCaptionFile,
+  isVTTFileValid,
+  isVideoCaptionLanguageValid
+}

server/core/helpers/custom-validators/video-channel-syncs.ts

@@ -0,0 +1,6 @@
+import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants.js'
+import { exists } from './misc.js'
+
+export function isVideoChannelSyncStateValid (value: any) {
+  return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined
+}

server/core/helpers/custom-validators/video-channels.ts

@@ -0,0 +1,32 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists } from './misc.js'
+import { isUserUsernameValid } from './users.js'
+
+const VIDEO_CHANNELS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_CHANNELS
+
+function isVideoChannelUsernameValid (value: string) {
+  // Use the same constraints than user username
+  return isUserUsernameValid(value)
+}
+
+function isVideoChannelDescriptionValid (value: string) {
+  return value === null || validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.DESCRIPTION)
+}
+
+function isVideoChannelDisplayNameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.NAME)
+}
+
+function isVideoChannelSupportValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.SUPPORT))
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoChannelUsernameValid,
+  isVideoChannelDescriptionValid,
+  isVideoChannelDisplayNameValid,
+  isVideoChannelSupportValid
+}

server/core/helpers/custom-validators/video-chapters.ts

@@ -0,0 +1,26 @@
+import { isArray } from './misc.js'
+import { VideoChapter, VideoChapterUpdate } from '@peertube/peertube-models'
+import { Unpacked } from '@peertube/peertube-typescript-utils'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import validator from 'validator'
+
+export function areVideoChaptersValid (value: VideoChapter[]) {
+  if (!isArray(value)) return false
+  if (!value.every(v => isVideoChapterValid(v))) return false
+
+  const timecodes = value.map(c => c.timecode)
+
+  return new Set(timecodes).size === timecodes.length
+}
+
+export function isVideoChapterValid (value: Unpacked<VideoChapterUpdate['chapters']>) {
+  return isVideoChapterTimecodeValid(value.timecode) && isVideoChapterTitleValid(value.title)
+}
+
+export function isVideoChapterTitleValid (value: any) {
+  return validator.default.isLength(value + '', CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE)
+}
+
+export function isVideoChapterTimecodeValid (value: any) {
+  return validator.default.isInt(value + '', { min: 0 })
+}

server/core/helpers/custom-validators/video-comments.ts

@@ -0,0 +1,14 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+
+const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS
+
+function isValidVideoCommentText (value: string) {
+  return value === null || validator.default.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidVideoCommentText
+}

server/core/helpers/custom-validators/video-imports.ts

@@ -0,0 +1,46 @@
+import 'multer'
+import { UploadFilesForCheck } from 'express'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants.js'
+import { exists, isFileValid } from './misc.js'
+
+function isVideoImportTargetUrlValid (url: string) {
+  const isURLOptions = {
+    require_host: true,
+    require_tld: true,
+    require_protocol: true,
+    require_valid_protocol: true,
+    protocols: [ 'http', 'https' ]
+  }
+
+  return exists(url) &&
+    validator.default.isURL('' + url, isURLOptions) &&
+    validator.default.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL)
+}
+
+function isVideoImportStateValid (value: any) {
+  return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined
+}
+
+// MacOS sends application/octet-stream
+const videoTorrentImportRegex = [ ...Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT), 'application/octet-stream' ]
+  .map(m => `(${m})`)
+  .join('|')
+
+function isVideoImportTorrentFile (files: UploadFilesForCheck) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: videoTorrentImportRegex,
+    field: 'torrentfile',
+    maxSize: CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max,
+    optional: true
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoImportStateValid,
+  isVideoImportTargetUrlValid,
+  isVideoImportTorrentFile
+}

server/core/helpers/custom-validators/video-lives.ts

@@ -0,0 +1,18 @@
+import { LiveVideoLatencyMode } from '@peertube/peertube-models'
+import { isDateValid } from './misc.js'
+
+export function isLiveLatencyModeValid (value: any) {
+  return [ LiveVideoLatencyMode.DEFAULT, LiveVideoLatencyMode.SMALL_LATENCY, LiveVideoLatencyMode.HIGH_LATENCY ].includes(value)
+}
+
+export function isLiveScheduleValid (schedule: any) {
+  return isDateValid(schedule?.startAt)
+}
+
+export function areLiveSchedulesValid (schedules: any[]) {
+  if (!schedules) return true
+
+  if (!Array.isArray(schedules)) return false
+
+  return schedules.every(schedule => isLiveScheduleValid(schedule))
+}

server/core/helpers/custom-validators/video-playlists.ts

@@ -0,0 +1,35 @@
+import { exists } from './misc.js'
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants.js'
+
+const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS
+
+function isVideoPlaylistNameValid (value: any) {
+  return exists(value) && validator.default.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.NAME)
+}
+
+function isVideoPlaylistDescriptionValid (value: any) {
+  return value === null || (exists(value) && validator.default.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.DESCRIPTION))
+}
+
+function isVideoPlaylistPrivacyValid (value: number) {
+  return validator.default.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined
+}
+
+function isVideoPlaylistTimestampValid (value: any) {
+  return value === null || (exists(value) && validator.default.isInt('' + value, { min: 0 }))
+}
+
+function isVideoPlaylistTypeValid (value: any) {
+  return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoPlaylistNameValid,
+  isVideoPlaylistDescriptionValid,
+  isVideoPlaylistPrivacyValid,
+  isVideoPlaylistTimestampValid,
+  isVideoPlaylistTypeValid
+}

server/core/helpers/custom-validators/video-rates.ts

@@ -0,0 +1,5 @@
+function isRatingValid (value: any) {
+  return value === 'like' || value === 'dislike'
+}
+
+export { isRatingValid }

server/core/helpers/custom-validators/video-redundancies.ts

@@ -0,0 +1,12 @@
+import { exists } from './misc.js'
+
+function isVideoRedundancyTarget (value: any) {
+  return exists(value) &&
+    (value === 'my-videos' || value === 'remote-videos')
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isVideoRedundancyTarget
+}

server/core/helpers/custom-validators/video-stats.ts

@@ -0,0 +1,16 @@
+import { VideoStatsTimeserieMetric } from '@peertube/peertube-models'
+
+const validMetrics = new Set<VideoStatsTimeserieMetric>([
+  'viewers',
+  'aggregateWatchTime'
+])
+
+function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) {
+  return validMetrics.has(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidStatTimeserieMetric
+}

server/core/helpers/custom-validators/video-studio.ts

@@ -0,0 +1,53 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import { buildTaskFileFieldname } from '@server/lib/video-studio.js'
+import { VideoStudioTask } from '@peertube/peertube-models'
+import { isArray } from './misc.js'
+import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos.js'
+import { forceNumber } from '@peertube/peertube-core-utils'
+
+function isValidStudioTasksArray (tasks: any) {
+  if (!isArray(tasks)) return false
+
+  return tasks.length >= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.min &&
+    tasks.length <= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.max
+}
+
+function isStudioCutTaskValid (task: VideoStudioTask) {
+  if (task.name !== 'cut') return false
+  if (!task.options) return false
+
+  const { start, end } = task.options
+  if (!start && !end) return false
+
+  if (start && !validator.default.isInt(start + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false
+  if (end && !validator.default.isInt(end + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false
+
+  if (!start || !end) return true
+
+  return forceNumber(start) < forceNumber(end)
+}
+
+function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
+  const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file'))
+
+  return (task.name === 'add-intro' || task.name === 'add-outro') &&
+    file && isVideoFileMimeTypeValid([ file ], null)
+}
+
+function isStudioTaskAddWatermarkValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) {
+  const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file'))
+
+  return task.name === 'add-watermark' &&
+    file && isVideoImageValid([ file ], null, true)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidStudioTasksArray,
+
+  isStudioCutTaskValid,
+  isStudioTaskAddIntroOutroValid,
+  isStudioTaskAddWatermarkValid
+}

server/core/helpers/custom-validators/video-transcoding.ts

@@ -0,0 +1,5 @@
+import { exists } from './misc.js'
+
+export function isValidCreateTranscodingType (value: any) {
+  return exists(value) && (value === 'hls' || value === 'web-video')
+}

server/core/helpers/custom-validators/video-view.ts

@@ -0,0 +1,27 @@
+import { VideoStatsUserAgentDevice } from '@peertube/peertube-models'
+import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
+import validator from 'validator'
+import { exists } from './misc.js'
+
+export function isVideoTimeValid (value: number, videoDuration?: number) {
+  if (value < 0) return false
+  if (exists(videoDuration) && value > videoDuration) return false
+
+  return true
+}
+
+export function isVideoViewEvent (value: string) {
+  return value === 'seek'
+}
+
+export function isVideoViewUAInfo (value: string) {
+  return validator.default.isLength(value, CONSTRAINTS_FIELDS.VIDEO_VIEW.UA_INFO)
+}
+
+// See https://docs.uaparser.dev/info/device/type.html
+const devices = new Set<VideoStatsUserAgentDevice>([ 'console', 'embedded', 'mobile', 'smarttv', 'tablet', 'wearable', 'xr', 'desktop' ])
+export function toVideoViewUADeviceOrNull (value: VideoStatsUserAgentDevice) {
+  return devices.has(value)
+    ? value
+    : null
+}

server/core/helpers/custom-validators/videos.ts

@@ -0,0 +1,205 @@
+import { HttpStatusCode, VideoIncludeType, VideoPrivacy, VideoPrivacyType, VideoRateType } from '@peertube/peertube-models'
+import { getVideoWithAttributes } from '@server/helpers/video.js'
+import { Request, Response, UploadFilesForCheck } from 'express'
+import { decode as magnetUriDecode } from 'magnet-uri'
+import validator from 'validator'
+import {
+  CONSTRAINTS_FIELDS,
+  MIMETYPES,
+  VIDEO_CATEGORIES,
+  VIDEO_COMMENTS_POLICY,
+  VIDEO_LICENCES,
+  VIDEO_LIVE,
+  VIDEO_PRIVACIES,
+  VIDEO_RATE_TYPES,
+  VIDEO_STATES
+} from '../../initializers/constants.js'
+import { exists, isArray, isDateValid, isFileValid } from './misc.js'
+
+const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
+
+export function isVideoIncludeValid (include: VideoIncludeType) {
+  return exists(include) && validator.default.isInt('' + include)
+}
+
+export function isVideoCategoryValid (value: any) {
+  return value === null || VIDEO_CATEGORIES[value] !== undefined
+}
+
+export function isVideoStateValid (value: any) {
+  return exists(value) && VIDEO_STATES[value] !== undefined
+}
+
+export function isVideoLicenceValid (value: any) {
+  return value === null || VIDEO_LICENCES[value] !== undefined
+}
+
+export function isVideoLanguageValid (value: any) {
+  return value === null ||
+    (typeof value === 'string' && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.LANGUAGE))
+}
+
+export function isVideoDurationValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION)
+}
+
+export function isVideoDescriptionValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION))
+}
+
+export function isVideoCommentsPolicyValid (value: any) {
+  return value === null || VIDEO_COMMENTS_POLICY[value] !== undefined
+}
+
+export function isVideoSupportValid (value: string) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.SUPPORT))
+}
+
+export function isVideoNameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME)
+}
+
+export function isVideoSourceFilenameValid (value: string) {
+  return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.VIDEO_SOURCE.FILENAME)
+}
+
+export function isVideoTagValid (tag: string) {
+  return exists(tag) && validator.default.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG)
+}
+
+export function areVideoTagsValid (tags: string[]) {
+  return tags === null || (
+    isArray(tags) &&
+    validator.default.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) &&
+    tags.every(tag => isVideoTagValid(tag))
+  )
+}
+
+export function isVideoViewsValid (value: string | number) {
+  return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS)
+}
+
+const ratingTypes = new Set(Object.values(VIDEO_RATE_TYPES))
+export function isVideoRatingTypeValid (value: string) {
+  return value === 'none' || ratingTypes.has(value as VideoRateType)
+}
+
+export function isVideoFileExtnameValid (value: string) {
+  return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
+}
+
+export function isVideoFileMimeTypeValid (files: UploadFilesForCheck, field = 'videofile') {
+  return isFileValid({
+    files,
+    mimeTypeRegex: MIMETYPES.VIDEO.MIMETYPES_REGEX,
+    field,
+    maxSize: null
+  })
+}
+
+const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME
+  .map(v => v.replace('.', ''))
+  .join('|')
+const videoImageTypesRegex = `image/(${videoImageTypes})`
+
+export function isVideoImageValid (files: UploadFilesForCheck, field: string, optional = true) {
+  return isFileValid({
+    files,
+    mimeTypeRegex: videoImageTypesRegex,
+    field,
+    maxSize: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max,
+    optional
+  })
+}
+
+export function isVideoPrivacyValid (value: number) {
+  return VIDEO_PRIVACIES[value] !== undefined
+}
+
+export function isVideoReplayPrivacyValid (value: number) {
+  return VIDEO_PRIVACIES[value] !== undefined && value !== VideoPrivacy.PASSWORD_PROTECTED
+}
+
+export function isScheduleVideoUpdatePrivacyValid (value: number) {
+  return value === VideoPrivacy.UNLISTED || value === VideoPrivacy.PUBLIC || value === VideoPrivacy.INTERNAL
+}
+
+export function isVideoOriginallyPublishedAtValid (value: string | null) {
+  return value === null || isDateValid(value)
+}
+
+export function isVideoFileInfoHashValid (value: string | null | undefined) {
+  return exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH)
+}
+
+export function isVideoFileResolutionValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '')
+}
+
+export function isVideoFPSResolutionValid (value: string) {
+  return value === null || validator.default.isInt(value + '')
+}
+
+export function isVideoFileSizeValid (value: string) {
+  return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE)
+}
+
+export function isVideoMagnetUriValid (value: string) {
+  if (!exists(value)) return false
+
+  const parsed = magnetUriDecode(value)
+  return parsed && isVideoFileInfoHashValid(parsed.infoHash)
+}
+
+export function isPasswordValid (password: string) {
+  return password.length >= CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.min &&
+    password.length < CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.max
+}
+
+export function isValidPasswordProtectedPrivacy (req: Request, res: Response) {
+  const fail = (message: string) => {
+    res.fail({
+      status: HttpStatusCode.BAD_REQUEST_400,
+      message
+    })
+    return false
+  }
+
+  let privacy: VideoPrivacyType
+  const video = getVideoWithAttributes(res)
+
+  if (exists(req.body?.privacy)) privacy = req.body.privacy
+  else if (exists(video?.privacy)) privacy = video.privacy
+
+  if (privacy !== VideoPrivacy.PASSWORD_PROTECTED) return true
+
+  if (!exists(req.body.videoPasswords) && !exists(req.body.passwords) && !exists(req.body.password)) {
+    return fail('Video passwords are missing.')
+  }
+
+  const passwords = req.body.videoPasswords || req.body.passwords || [ req.body.password ]
+
+  if (passwords.length === 0) return fail('At least one video password is required.')
+
+  if (new Set(passwords).size !== passwords.length) return fail('Duplicate video passwords are not allowed.')
+
+  for (const password of passwords) {
+    if (typeof password !== 'string') {
+      return fail('Video password should be a string.')
+    }
+
+    if (!isPasswordValid(password)) {
+      return fail('Invalid video password. Password length should be at least 2 characters and no more than 100 characters.')
+    }
+  }
+
+  return true
+}
+
+export function isNSFWFlagsValid (value: number) {
+  return value === null || (exists(value) && validator.default.isInt('' + value))
+}
+
+export function isNSFWSummaryValid (value: any) {
+  return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NSFW_SUMMARY))
+}

server/core/helpers/custom-validators/watched-words.ts

@@ -0,0 +1,17 @@
+import validator from 'validator'
+import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js'
+import { exists, isArray } from './misc.js'
+
+export function isWatchedWordListNameValid (listName: string) {
+  return exists(listName) && validator.default.isLength(listName, CONSTRAINTS_FIELDS.WATCHED_WORDS.LIST_NAME)
+}
+
+export function isWatchedWordValid (word: string) {
+  return exists(word) && validator.default.isLength(word, CONSTRAINTS_FIELDS.WATCHED_WORDS.WORD)
+}
+
+export function areWatchedWordsValid (words: string[]) {
+  return isArray(words) &&
+    validator.default.isInt(words.length.toString(), CONSTRAINTS_FIELDS.WATCHED_WORDS.WORDS) &&
+    words.every(word => isWatchedWordValid(word))
+}

server/core/helpers/custom-validators/webfinger.ts

@@ -0,0 +1,15 @@
+import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants.js'
+import { sanitizeHost } from '../core-utils.js'
+import { exists } from './misc.js'
+
+export function isWebfingerLocalResourceValid (value: string) {
+  if (!exists(value)) return false
+  if (value.startsWith('acct:') === false) return false
+
+  const actorWithHost = value.substring(5)
+  const actorParts = actorWithHost.split('@')
+  if (actorParts.length !== 2) return false
+
+  const host = actorParts[1]
+  return sanitizeHost(host, REMOTE_SCHEME.HTTP) === WEBSERVER.HOST
+}

server/core/helpers/database-utils.ts

@@ -0,0 +1,127 @@
+import retry from 'async/retry.js'
+import Bluebird from 'bluebird'
+import { Transaction } from 'sequelize'
+import { Model } from 'sequelize-typescript'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { logger } from './logger.js'
+
+export function retryTransactionWrapper<T, A, B, C, D> (
+  functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T>,
+  arg1: A,
+  arg2: B,
+  arg3: C,
+  arg4: D
+): Promise<T>
+
+export function retryTransactionWrapper<T, A, B, C> (
+  functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T>,
+  arg1: A,
+  arg2: B,
+  arg3: C
+): Promise<T>
+
+export function retryTransactionWrapper<T, A, B> (
+  functionToRetry: (arg1: A, arg2: B) => Promise<T>,
+  arg1: A,
+  arg2: B
+): Promise<T>
+
+export function retryTransactionWrapper<T, A> (
+  functionToRetry: (arg1: A) => Promise<T>,
+  arg1: A
+): Promise<T>
+
+export function retryTransactionWrapper<T> (
+  functionToRetry: () => Promise<T> | Bluebird<T>
+): Promise<T>
+
+export function retryTransactionWrapper<T> (
+  functionToRetry: (...args: any[]) => Promise<T>,
+  ...args: any[]
+): Promise<T> {
+  return transactionRetryer<T>(callback => {
+    functionToRetry.apply(null, args)
+      .then((result: T) => callback(null, result))
+      .catch(err => callback(err))
+  })
+    .catch(err => {
+      logger.warn(`Cannot execute ${functionToRetry.name || 'function'} with many retries.`, { err })
+      throw err
+    })
+}
+
+export function transactionRetryer<T> (func: (err: any, data: T) => any) {
+  return new Promise<T>((res, rej) => {
+    retry(
+      {
+        times: 5,
+
+        errorFilter: err => {
+          const willRetry = err.name === 'SequelizeDatabaseError'
+          logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] })
+          return willRetry
+        }
+      },
+      func,
+      (err, data) => err ? rej(err) : res(data)
+    )
+  })
+}
+
+export function saveInTransactionWithRetries<T extends Pick<Model, 'save' | 'changed'>> (model: T) {
+  const changedKeys = model.changed() || []
+
+  return retryTransactionWrapper(() => {
+    return sequelizeTypescript.transaction(async transaction => {
+      try {
+        await model.save({ transaction })
+      } catch (err) {
+        // Reinit changed keys
+        for (const key of changedKeys) {
+          model.changed(key as keyof Model, true)
+        }
+
+        throw err
+      }
+    })
+  })
+}
+
+export function deleteInTransactionWithRetries<T extends Pick<Model, 'destroy'>> (model: T) {
+  return retryTransactionWrapper(() => {
+    return sequelizeTypescript.transaction(async transaction => {
+      await model.destroy({ transaction })
+    })
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export function resetSequelizeInstance<T> (instance: Model<T>) {
+  return instance.reload()
+}
+
+export function filterNonExistingModels<T extends { hasSameUniqueKeysThan(other: T): boolean }> (
+  fromDatabase: T[],
+  newModels: T[]
+) {
+  return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f)))
+}
+
+export function deleteAllModels<T extends Pick<Model, 'destroy'>> (models: T[], transaction: Transaction) {
+  return Promise.all(models.map(f => f.destroy({ transaction })))
+}
+
+// ---------------------------------------------------------------------------
+
+export function runInReadCommittedTransaction<T> (fn: (t: Transaction) => Promise<T>) {
+  const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED }
+
+  return sequelizeTypescript.transaction(options, t => fn(t))
+}
+
+export function afterCommitIfTransaction (t: Transaction, fn: Function) {
+  if (t) return t.afterCommit(() => fn())
+
+  return fn()
+}

server/core/helpers/debounce.ts

@@ -0,0 +1,15 @@
+export function Debounce (config: { timeoutMS: number }) {
+  let timeoutRef: NodeJS.Timeout
+
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const original = descriptor.value
+
+    descriptor.value = function (...args: any[]) {
+      clearTimeout(timeoutRef)
+
+      timeoutRef = setTimeout(() => {
+        original.apply(this, args)
+      }, config.timeoutMS)
+    }
+  }
+}

server/core/helpers/decache.ts

@@ -0,0 +1,78 @@
+// Thanks: https://github.com/dwyl/decache
+// We reuse this file to also uncache plugin base path
+
+import { CONFIG } from '@server/initializers/config.js'
+import { Module } from 'module'
+import { extname, join } from 'path'
+
+export function decachePlugin (require: NodeJS.Require, libraryPath: string) {
+  const moduleName = find(require, libraryPath)
+
+  if (!moduleName) return
+
+  searchCache(require, moduleName, function (mod) {
+    console.log(mod)
+
+    delete require.cache[mod.id]
+
+    removeCachedPath(mod.path)
+    removeCachedPath(join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules'))
+  })
+}
+
+export function decacheModule (require: NodeJS.Require, name: string) {
+  const moduleName = find(require, name)
+
+  if (!moduleName) return
+
+  searchCache(require, moduleName, function (mod) {
+    delete require.cache[mod.id]
+
+    removeCachedPath(mod.path)
+  })
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function find (require: NodeJS.Require, moduleName: string) {
+  try {
+    return require.resolve(moduleName)
+  } catch {
+    return ''
+  }
+}
+
+function searchCache (require: NodeJS.Require, moduleName: string, callback: (current: NodeJS.Module) => void) {
+  const resolvedModule = require.resolve(moduleName)
+  let mod: NodeJS.Module
+  const visited = {}
+
+  if (resolvedModule && ((mod = require.cache[resolvedModule]) !== undefined)) {
+    // Recursively go over the results
+    ;(function run (current) {
+      visited[current.id] = true
+
+      current.children.forEach(function (child) {
+        if (extname(child.filename) !== '.node' && !visited[child.id]) {
+          run(child)
+        }
+      })
+
+      // Call the specified callback providing the
+      // found module
+      callback(current)
+    })(mod)
+  }
+}
+
+function removeCachedPath (pluginPath: string) {
+  const pathCache = (Module as any)._pathCache as { [id: string]: string[] }
+
+  Object.keys(pathCache).forEach(function (cacheKey) {
+    if (cacheKey.includes(pluginPath)) {
+      delete pathCache[cacheKey]
+    }
+  })
+}

server/core/helpers/dns.ts

@@ -0,0 +1,29 @@
+import { lookup } from 'dns'
+import ipaddr from 'ipaddr.js'
+
+function dnsLookupAll (hostname: string) {
+  return new Promise<string[]>((res, rej) => {
+    lookup(hostname, { family: 0, all: true }, (err, adresses) => {
+      if (err) return rej(err)
+
+      return res(adresses.map(a => a.address))
+    })
+  })
+}
+
+async function isResolvingToUnicastOnly (hostname: string) {
+  const addresses = await dnsLookupAll(hostname)
+
+  for (const address of addresses) {
+    const parsed = ipaddr.parse(address)
+
+    if (parsed.range() !== 'unicast') return false
+  }
+
+  return true
+}
+
+export {
+  dnsLookupAll,
+  isResolvingToUnicastOnly
+}

server/core/helpers/express-utils.ts

@@ -0,0 +1,200 @@
+import { NSFWFlag, VideosCommonQuery } from '@peertube/peertube-models'
+import { getLowercaseExtension } from '@peertube/peertube-node-utils'
+import express, { RequestHandler } from 'express'
+import multer, { diskStorage } from 'multer'
+import { CONFIG } from '../initializers/config.js'
+import { REMOTE_SCHEME } from '../initializers/constants.js'
+import { isArray } from './custom-validators/misc.js'
+import { logger } from './logger.js'
+import { deleteFileAndCatch, generateRandomString } from './utils.js'
+import { getExtFromMimetype } from './video.js'
+
+// ---------------------------------------------------------------------------
+// Extract NSFW Filters options to list videos
+// ---------------------------------------------------------------------------
+
+export function buildNSFWFilters (options: {
+  req?: express.Request
+  res?: express.Response
+} = {}) {
+  return {
+    nsfw: buildNSFWFilter(options),
+
+    nsfwFlagsIncluded: CONFIG.NSFW_FLAGS_SETTINGS.ENABLED
+      ? buildNSFWFlagsIncluded(options)
+      : NSFWFlag.NONE,
+
+    nsfwFlagsExcluded: CONFIG.NSFW_FLAGS_SETTINGS.ENABLED
+      ? buildNSFWFlagsExcluded(options)
+      : NSFWFlag.NONE
+  }
+}
+
+function buildNSFWFilter (options: {
+  req?: express.Request
+  res?: express.Response
+}) {
+  const query = options.req?.query.nsfw as VideosCommonQuery['nsfw']
+  const user = options.res?.locals.oauth?.token.User
+
+  if (query === 'true') return true
+  if (query === 'false') return false
+  if (query === 'both') return undefined
+
+  if (user) {
+    // User does not want NSFW videos
+    if (user.nsfwPolicy === 'do_not_list') return false
+
+    // Both
+    return undefined
+  }
+
+  if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false
+
+  // Display all
+  return null
+}
+
+function buildNSFWFlagsIncluded (options: {
+  req?: express.Request
+  res?: express.Response
+}) {
+  const query = options.req?.query.nsfwFlagsIncluded as VideosCommonQuery['nsfwFlagsIncluded']
+  const user = options.res?.locals.oauth?.token.User
+
+  if (query) return query
+  if (user) return user.nsfwFlagsWarned | user.nsfwFlagsBlurred | user.nsfwFlagsDisplayed
+
+  return undefined
+}
+
+function buildNSFWFlagsExcluded (options: {
+  req?: express.Request
+  res?: express.Response
+}) {
+  const query = options.req?.query.nsfwFlagsExcluded as VideosCommonQuery['nsfwFlagsExcluded']
+  const user = options.res?.locals.oauth?.token.User
+
+  if (query) return query
+  if (user) return user.nsfwFlagsHidden
+
+  return undefined
+}
+
+// ---------------------------------------------------------------------------
+
+export function cleanUpReqFiles (req: express.Request) {
+  const filesObject = req.files
+  if (!filesObject) return
+
+  if (isArray(filesObject)) {
+    filesObject.forEach(f => deleteFileAndCatch(f.path))
+    return
+  }
+
+  for (const key of Object.keys(filesObject)) {
+    const files = filesObject[key]
+
+    files.forEach(f => deleteFileAndCatch(f.path))
+  }
+}
+
+export function getHostWithPort (host: string) {
+  const splitted = host.split(':')
+
+  // The port was not specified
+  if (splitted.length === 1) {
+    if (REMOTE_SCHEME.HTTP === 'https') return host + ':443'
+
+    return host + ':80'
+  }
+
+  return host
+}
+
+export function createReqFiles (
+  fieldNames: string[],
+  mimeTypes: { [id: string]: string | string[] },
+  destination = CONFIG.STORAGE.TMP_DIR
+): RequestHandler {
+  const storage = diskStorage({
+    destination: (req, file, cb) => {
+      cb(null, destination)
+    },
+
+    filename: (req, file, cb) => {
+      return generateReqFilename(file, mimeTypes, cb)
+    }
+  })
+
+  const fields: { name: string, maxCount: number }[] = []
+  for (const fieldName of fieldNames) {
+    fields.push({
+      name: fieldName,
+      maxCount: 1
+    })
+  }
+
+  return multer({ storage }).fields(fields)
+}
+
+export function createAnyReqFiles (
+  mimeTypes: { [id: string]: string | string[] },
+  fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void
+): RequestHandler {
+  const storage = diskStorage({
+    destination: (req, file, cb) => {
+      cb(null, CONFIG.STORAGE.TMP_DIR)
+    },
+
+    filename: (req, file, cb) => {
+      return generateReqFilename(file, mimeTypes, cb)
+    }
+  })
+
+  return multer({ storage, fileFilter }).any()
+}
+
+export function isUserAbleToSearchRemoteURI (res: express.Response) {
+  const user = res.locals.oauth ? res.locals.oauth.token.User : undefined
+
+  return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true ||
+    (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined)
+}
+
+export function getCountVideos (req: express.Request) {
+  return req.query.skipCount !== true
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function generateReqFilename (
+  file: Express.Multer.File,
+  mimeTypes: { [id: string]: string | string[] },
+  cb: (err: Error, name: string) => void
+) {
+  let extension: string
+  const fileExtension = getLowercaseExtension(file.originalname)
+  const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype)
+
+  // Take the file extension if we don't understand the mime type
+  if (!extensionFromMimetype) {
+    extension = fileExtension
+  } else {
+    // Take the first available extension for this mimetype
+    extension = extensionFromMimetype
+  }
+
+  let randomString = ''
+
+  try {
+    randomString = await generateRandomString(16)
+  } catch (err) {
+    logger.error('Cannot generate random string for file name.', { err })
+    randomString = 'fake-random-string'
+  }
+
+  cb(null, randomString + extension)
+}

server/core/helpers/ffmpeg/codecs.ts

@@ -0,0 +1,64 @@
+import { FfprobeData } from 'fluent-ffmpeg'
+import { getAudioStream, getVideoStream } from '@peertube/peertube-ffmpeg'
+import { logger } from '../logger.js'
+import { forceNumber } from '@peertube/peertube-core-utils'
+
+export async function getVideoStreamCodec (path: string, existingProbe?: FfprobeData) {
+  const videoStream = await getVideoStream(path, existingProbe)
+  if (!videoStream) return ''
+
+  const videoCodec = videoStream.codec_tag_string
+
+  if (videoCodec === 'vp09') return 'vp09.00.50.08'
+  if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0'
+
+  const baseProfileMatrix = {
+    avc1: {
+      High: '6400',
+      Main: '4D40',
+      Baseline: '42E0'
+    },
+    av01: {
+      High: '1',
+      Main: '0',
+      Professional: '2'
+    }
+  }
+
+  let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile]
+  if (!baseProfile) {
+    logger.warn('Cannot get video profile codec of %s.', path, { videoStream })
+    baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback
+  }
+
+  if (videoCodec === 'av01') {
+    let level = videoStream.level.toString()
+    if (level.length === 1) level = `0${level}`
+
+    // Guess the tier indicator and bit depth
+    return `${videoCodec}.${baseProfile}.${level}M.08`
+  }
+
+  let level = forceNumber(videoStream.level).toString(16)
+  if (level.length === 1) level = `0${level}`
+
+  // Default, h264 codec
+  return `${videoCodec}.${baseProfile}${level}`
+}
+
+export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) {
+  const { audioStream } = await getAudioStream(path, existingProbe)
+
+  if (!audioStream) return ''
+
+  const audioCodecName = audioStream.codec_name
+
+  if (audioCodecName === 'opus') return 'opus'
+  if (audioCodecName === 'vorbis') return 'vorbis'
+  if (audioCodecName === 'aac') return 'mp4a.40.2'
+  if (audioCodecName === 'mp3') return 'mp4a.40.34'
+
+  logger.warn('Cannot get audio codec of %s.', path, { audioStream })
+
+  return 'mp4a.40.2' // Fallback
+}

server/core/helpers/ffmpeg/ffmpeg-image.ts

@@ -0,0 +1,10 @@
+import { FFmpegImage } from '@peertube/peertube-ffmpeg'
+import { getFFmpegCommandWrapperOptions } from './ffmpeg-options.js'
+
+export function processImage (options: Parameters<FFmpegImage['processImage']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processImage(options)
+}
+
+export function generateThumbnailFromVideo (options: Parameters<FFmpegImage['generateThumbnailFromVideo']>[0]) {
+  return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options)
+}

server/core/helpers/ffmpeg/ffmpeg-options.ts

@@ -0,0 +1,45 @@
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { FFMPEG_NICE } from '@server/initializers/constants.js'
+import { FFmpegCommandWrapperOptions } from '@peertube/peertube-ffmpeg'
+import { AvailableEncoders } from '@peertube/peertube-models'
+
+type CommandType = 'live' | 'vod' | 'thumbnail'
+
+export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions {
+  return {
+    availableEncoders,
+    profile: getProfile(type),
+
+    niceness: FFMPEG_NICE[type.toUpperCase()],
+    tmpDirectory: CONFIG.STORAGE.TMP_DIR,
+    threads: getThreads(type),
+
+    logger: {
+      debug: logger.debug.bind(logger),
+      info: logger.info.bind(logger),
+      warn: logger.warn.bind(logger),
+      error: logger.error.bind(logger)
+    },
+    lTags: { tags: [ 'ffmpeg' ] }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getThreads (type: CommandType) {
+  if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS
+  if (type === 'vod') return CONFIG.TRANSCODING.THREADS
+
+  // Auto
+  return 0
+}
+
+function getProfile (type: CommandType) {
+  if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE
+  if (type === 'vod') return CONFIG.TRANSCODING.PROFILE
+
+  return undefined
+}

server/core/helpers/ffmpeg/framerate.ts

@@ -0,0 +1,93 @@
+import { VideoResolution } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { logger } from '../logger.js'
+
+export function computeOutputFPS (options: {
+  inputFPS: number
+  isOriginResolution: boolean
+  resolution: number
+  type: 'vod' | 'live'
+}) {
+  const { resolution, isOriginResolution, type } = options
+
+  if (resolution === VideoResolution.H_NOVIDEO) return 0
+
+  const settings = type === 'vod'
+    ? buildTranscodingFPSOptions(CONFIG.TRANSCODING.FPS.MAX)
+    : buildTranscodingFPSOptions(CONFIG.LIVE.TRANSCODING.FPS.MAX)
+
+  let fps = options.inputFPS
+
+  if (
+    // On small/medium transcoded resolutions, limit FPS
+    !isOriginResolution &&
+    resolution !== undefined &&
+    resolution < settings.KEEP_ORIGIN_FPS_RESOLUTION_MIN &&
+    fps > settings.AVERAGE
+  ) {
+    // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value
+    fps = getClosestFramerate({ fps, settings, type: 'STANDARD' })
+  }
+
+  if (fps < settings.HARD_MIN) {
+    throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${settings.HARD_MIN}`)
+  }
+
+  // Cap min FPS
+  fps = Math.max(fps, settings.TRANSCODED_MIN)
+
+  // Cap max FPS for non-origin video file
+  if (!isOriginResolution && fps > settings.TRANSCODED_MAX) {
+    fps = getClosestFramerate({ fps, settings, type: 'HD_STANDARD' })
+  }
+
+  logger.debug(`Computed output FPS ${fps} for resolution ${resolution}p`, { options, settings })
+
+  return fps
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function buildTranscodingFPSOptions (maxFPS: number) {
+  const STANDARD = [ 24, 25, 30 ].filter(v => v <= maxFPS)
+  if (STANDARD.length === 0) STANDARD.push(maxFPS)
+
+  const HD_STANDARD = [ 50, 60, maxFPS ].filter(v => v <= maxFPS)
+
+  return {
+    HARD_MIN: 0.1,
+
+    TRANSCODED_MIN: 1,
+
+    TRANSCODED_MAX: maxFPS,
+
+    STANDARD,
+    HD_STANDARD,
+
+    AVERAGE: Math.min(30, maxFPS),
+
+    KEEP_ORIGIN_FPS_RESOLUTION_MIN: 720 // We keep the original FPS on high resolutions (720 minimum)
+  }
+}
+
+function getClosestFramerate (options: {
+  fps: number
+  settings: ReturnType<typeof buildTranscodingFPSOptions>
+  type: Extract<keyof ReturnType<typeof buildTranscodingFPSOptions>, 'HD_STANDARD' | 'STANDARD'>
+}) {
+  const { fps, settings, type } = options
+
+  const copy = [ ...settings[type] ]
+
+  // Biggest FPS first
+  const descSorted = copy.sort((a, b) => b - a)
+  // Find biggest FPS that can be divided by input FPS
+  const found = descSorted.find(e => fps % e === 0)
+
+  if (found) return found
+
+  // Approximation to the best result
+  return copy.sort((a, b) => fps % a - fps % b)[0]
+}

server/core/helpers/ffmpeg/index.ts

@@ -0,0 +1,4 @@
+export * from './codecs.js'
+export * from './ffmpeg-image.js'
+export * from './ffmpeg-options.js'
+export * from './framerate.js'

server/core/helpers/fs.ts

@@ -0,0 +1,12 @@
+import { move } from 'fs-extra/esm'
+import { rename } from 'fs/promises'
+
+export async function tryAtomicMove (src: string, destination: string) {
+  try {
+    await rename(src, destination)
+  } catch (err) {
+    if (err?.code !== 'EXDEV') throw err
+
+    return move(src, destination, { overwrite: true })
+  }
+}

server/core/helpers/geo-ip.ts

@@ -0,0 +1,155 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { pathExists } from 'fs-extra/esm'
+import { writeFile } from 'fs/promises'
+import maxmind, { CityResponse, CountryResponse, Reader } from 'maxmind'
+import { join } from 'path'
+import { isArray } from './custom-validators/misc.js'
+import { logger, loggerTagsFactory } from './logger.js'
+import { isBinaryResponse, unsafeSSRFGot } from './requests.js'
+
+const lTags = loggerTagsFactory('geo-ip')
+
+export class GeoIP {
+  private static instance: GeoIP
+
+  private countryReader: Reader<CountryResponse>
+  private cityReader: Reader<CityResponse>
+
+  private lastInitTry: Date
+  private initReadersPromise: Promise<any>
+
+  private readonly INIT_READERS_RETRY_INTERVAL = 1000 * 60 * 10 // 10 minutes
+  private readonly countryDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-country-lite-latest.mmdb')
+  private readonly cityDBPath = join(CONFIG.STORAGE.BIN_DIR, 'dbip-city-lite-latest.mmdb')
+
+  private constructor () {
+  }
+
+  async safeIPISOLookup (ip: string): Promise<{ country: string, subdivisionName: string }> {
+    const emptyResult = { country: null, subdivisionName: null }
+    if (CONFIG.GEO_IP.ENABLED === false) return emptyResult
+
+    try {
+      if (this.initReadersPromise === undefined) {
+        this.initReadersPromise = this.initReadersIfNeeded()
+      }
+
+      await this.initReadersPromise
+      this.initReadersPromise = undefined
+
+      const countryResult = this.countryReader?.get(ip)
+      const cityResult = this.cityReader?.get(ip)
+
+      return {
+        country: this.getISOCountry(countryResult),
+        subdivisionName: this.getISOSubdivision(cityResult)
+      }
+    } catch (err) {
+      logger.error('Cannot get country/city information from IP.', { err })
+
+      return emptyResult
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private getISOCountry (countryResult: CountryResponse) {
+    return countryResult?.country?.iso_code || null
+  }
+
+  private getISOSubdivision (subdivisionResult: CityResponse) {
+    const subdivisions = subdivisionResult?.subdivisions
+    if (!isArray(subdivisions) || subdivisions.length === 0) return null
+
+    // The last subdivision is the more precise one
+    const subdivision = subdivisions[subdivisions.length - 1]
+
+    return subdivision.names?.en || null
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async updateDatabases () {
+    if (CONFIG.GEO_IP.ENABLED === false) return
+
+    await this.updateCountryDatabase()
+    await this.updateCityDatabase()
+  }
+
+  private async updateCountryDatabase () {
+    if (!CONFIG.GEO_IP.COUNTRY.DATABASE_URL) return false
+
+    await this.updateDatabaseFile(CONFIG.GEO_IP.COUNTRY.DATABASE_URL, this.countryDBPath)
+
+    this.countryReader = undefined
+
+    return true
+  }
+
+  private async updateCityDatabase () {
+    if (!CONFIG.GEO_IP.CITY.DATABASE_URL) return false
+
+    await this.updateDatabaseFile(CONFIG.GEO_IP.CITY.DATABASE_URL, this.cityDBPath)
+
+    this.cityReader = undefined
+
+    return true
+  }
+
+  private async updateDatabaseFile (url: string, destination: string) {
+    logger.info('Updating GeoIP databases from %s.', url, lTags())
+
+    const gotOptions = { context: { bodyKBLimit: 800_000 }, responseType: 'buffer' as 'buffer' }
+
+    try {
+      const gotResult = await unsafeSSRFGot(url, gotOptions)
+
+      if (!isBinaryResponse(gotResult)) {
+        throw new Error('Not a binary response')
+      }
+
+      await writeFile(destination, gotResult.body)
+
+      logger.info('GeoIP database updated %s.', destination, lTags())
+    } catch (err) {
+      logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async initReadersIfNeeded () {
+    if (this.lastInitTry && this.lastInitTry.getTime() > Date.now() - this.INIT_READERS_RETRY_INTERVAL) return
+    this.lastInitTry = new Date()
+
+    if (!this.countryReader) {
+      let open = true
+
+      if (!await pathExists(this.countryDBPath)) {
+        open = await this.updateCountryDatabase()
+      }
+
+      if (open) {
+        this.countryReader = await maxmind.open(this.countryDBPath)
+      }
+    }
+
+    if (!this.cityReader) {
+      let open = true
+
+      if (!await pathExists(this.cityDBPath)) {
+        open = await this.updateCityDatabase()
+      }
+
+      if (open) {
+        this.cityReader = await maxmind.open(this.cityDBPath)
+      }
+    }
+  }
+
+  // ---------------------------------------------------------------------------
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/helpers/hpagent.ts

@@ -0,0 +1,158 @@
+// Copy of un-maintained hpagent package with https://github.com/delvedor/hpagent/pull/114 fix
+
+import http from 'http'
+import https from 'https'
+import { Socket, TcpNetConnectOpts } from 'net'
+import { URL } from 'url'
+
+type Options = {
+  keepAlive: boolean
+  keepAliveMsecs: number
+  maxSockets: number
+  maxFreeSockets: number
+  scheduling: 'lifo'
+  proxy: string
+}
+
+export class HttpProxyAgent extends http.Agent {
+  private readonly proxy: URL
+  private readonly keepAlive: boolean
+
+  constructor (options: Options) {
+    const { proxy, ...opts } = options
+
+    super(opts)
+
+    this.keepAlive = options.keepAlive
+
+    this.proxy = typeof proxy === 'string'
+      ? new URL(proxy)
+      : proxy
+  }
+
+  createConnection (options: TcpNetConnectOpts, callback?: (err: Error, socket: Socket) => void) {
+    const requestOptions = {
+      method: 'CONNECT',
+      host: this.proxy.hostname,
+      port: this.proxy.port,
+      path: `${options.host}:${options.port}`,
+      setHost: false,
+      headers: { connection: this.keepAlive ? 'keep-alive' : 'close', host: `${options.host}:${options.port}` },
+      agent: false,
+      timeout: options.timeout || 0,
+      servername: undefined as string
+    }
+
+    if (this.proxy.username || this.proxy.password) {
+      const base64 = Buffer.from(
+        `${decodeURIComponent(this.proxy.username || '')}:${decodeURIComponent(this.proxy.password || '')}`
+      ).toString('base64')
+
+      requestOptions.headers['proxy-authorization'] = `Basic ${base64}`
+    }
+
+    if (this.proxy.protocol === 'https:') {
+      requestOptions.servername = this.proxy.hostname
+    }
+
+    const request = (this.proxy.protocol === 'http:' ? http : https).request(requestOptions)
+    request.once('connect', (response, socket, head) => {
+      request.removeAllListeners()
+      socket.removeAllListeners()
+      if (response.statusCode === 200) {
+        callback(null, socket)
+      } else {
+        socket.destroy()
+        callback(new Error(`Bad response: ${response.statusCode}`), null)
+      }
+    })
+
+    request.once('timeout', () => {
+      request.destroy(new Error('Proxy timeout'))
+    })
+
+    request.once('error', err => {
+      request.removeAllListeners()
+      callback(err, null)
+    })
+
+    request.end()
+  }
+}
+
+export class HttpsProxyAgent extends https.Agent {
+  private readonly proxy: URL
+  private readonly keepAlive: boolean
+
+  constructor (options: Options) {
+    const { proxy, ...opts } = options
+
+    super(opts)
+
+    this.keepAlive = options.keepAlive
+
+    this.proxy = typeof proxy === 'string'
+      ? new URL(proxy)
+      : proxy
+  }
+
+  createConnection (options: TcpNetConnectOpts, callback?: (err: Error, socket: Socket) => void) {
+    const requestOptions = {
+      method: 'CONNECT',
+      host: this.proxy.hostname,
+      port: this.proxy.port,
+      path: `${options.host}:${options.port}`,
+      setHost: false,
+      headers: { connection: this.keepAlive ? 'keep-alive' : 'close', host: `${options.host}:${options.port}` },
+      agent: false,
+      timeout: options.timeout || 0,
+      servername: undefined as string
+    }
+
+    if (this.proxy.username || this.proxy.password) {
+      const base64 = Buffer.from(
+        `${decodeURIComponent(this.proxy.username || '')}:${decodeURIComponent(this.proxy.password || '')}
+      `).toString('base64')
+
+      requestOptions.headers['proxy-authorization'] = `Basic ${base64}`
+    }
+
+    // Necessary for the TLS check with the proxy to succeed.
+    if (this.proxy.protocol === 'https:') {
+      requestOptions.servername = this.proxy.hostname
+    }
+
+    const request = (this.proxy.protocol === 'http:' ? http : https).request(requestOptions)
+    request.once('connect', (response, socket, head) => {
+      request.removeAllListeners()
+      socket.removeAllListeners()
+
+      if (response.statusCode === 200) {
+        try {
+          // FIXME: typings doesn't include createConnection type in HTTP agent
+          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+          // @ts-expect-error
+          const secureSocket = super.createConnection({ ...options, socket })
+          callback(null, secureSocket)
+        } catch (err) {
+          socket.destroy()
+          callback(err, null)
+        }
+      } else {
+        socket.destroy()
+        callback(new Error(`Bad response: ${response.statusCode}`), null)
+      }
+    })
+
+    request.once('timeout', () => {
+      request.destroy(new Error('Proxy timeout'))
+    })
+
+    request.once('error', err => {
+      request.removeAllListeners()
+      callback(err, null)
+    })
+
+    request.end()
+  }
+}

server/core/helpers/i18n.ts

@@ -0,0 +1,98 @@
+import { AVAILABLE_LOCALES, getCompleteLocale } from '@peertube/peertube-core-utils'
+import { CONFIG } from '@server/initializers/config.js'
+import { LANGUAGE_COOKIE_NAME, LANGUAGE_HEADER, SERVER_INTERNAL_LOCALES_BASE_PATH } from '@server/initializers/constants.js'
+import { MUser } from '@server/types/models/index.js'
+import Bluebird from 'bluebird'
+import express from 'express'
+import { readJson } from 'fs-extra/esm'
+import { readdir } from 'fs/promises'
+import i18next from 'i18next'
+import ICU from 'i18next-icu'
+import { join } from 'path'
+import { logger } from './logger.js'
+
+export async function initI18n () {
+  const locales = await readdir(SERVER_INTERNAL_LOCALES_BASE_PATH)
+  const resources: Record<string, Record<string, string>> = {}
+
+  await Bluebird.map(locales, async locale => {
+    const localePath = join(SERVER_INTERNAL_LOCALES_BASE_PATH, locale)
+
+    const translation = await readJson(join(localePath, 'translation.json'))
+    resources[locale] = { translation }
+  }, { concurrency: 10 })
+
+  return i18next.use(ICU)
+    .init({
+      resources,
+      nsSeparator: false,
+      keySeparator: false,
+
+      // do not load a fallback
+      fallbackLng: false
+    })
+    .then(() => logger.info('i18n initialized with locales: ' + Object.keys(resources).join(', ')))
+}
+
+// ---------------------------------------------------------------------------
+
+export type TranslateFn = (key: string, context: Record<string, string | number>) => string
+
+export function useI18n (req: express.Request, res: express.Response, next: express.NextFunction) {
+  req.t = (key: string, context: Record<string, string | number> = {}) => {
+    // Use req special header language
+    // Or user language
+    // Or default language
+    const language = guessLanguageFromReq(req, res)
+
+    return t(key, language, context)
+  }
+
+  next()
+}
+
+export function guessLanguageFromReq (req: express.Request, res: express.Response) {
+  return req.headers[LANGUAGE_HEADER] as string ||
+    res.locals.oauth?.token?.User?.language ||
+    req.acceptsLanguages(AVAILABLE_LOCALES) ||
+    CONFIG.INSTANCE.DEFAULT_LANGUAGE
+}
+
+export function t (
+  key: string,
+  language: string,
+  context: Record<string, string | number> = {}
+) {
+  if (!language) throw new Error('Language is required for translation')
+
+  if (!i18next.isInitialized) {
+    logger.warn('i18next is not initialized, translation will not work')
+
+    return key
+  }
+
+  return i18next.t(key, { lng: getCompleteLocale(language), ...context })
+}
+
+export function tu (key: string, user: Pick<MUser, 'language' | 'getLanguage'>, context: Record<string, string | number> = {}) {
+  return t(key, user.getLanguage(), context)
+}
+
+// ---------------------------------------------------------------------------
+
+export function setClientLanguageCookie (res: express.Response, language: string) {
+  if (language === null) {
+    res.clearCookie(LANGUAGE_COOKIE_NAME)
+    return
+  }
+
+  res.cookie(LANGUAGE_COOKIE_NAME, language, {
+    secure: true,
+    sameSite: 'none',
+    maxAge: 1000 * 3600 * 24 * 90 // 3 months
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export const englishLanguage = 'en-US'

server/core/helpers/image-utils.ts

@@ -0,0 +1,171 @@
+import { ColorActionName } from '@jimp/plugin-color'
+import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils'
+import { copy, remove } from 'fs-extra/esm'
+import { readFile } from 'fs/promises'
+import { processImage as processImageByFFmpeg } from './ffmpeg/index.js'
+import { logger } from './logger.js'
+
+import type Jimp from 'jimp'
+
+export function generateImageFilename (extension = '.jpg') {
+  return buildUUID() + extension
+}
+
+export async function processImage (options: {
+  path: string
+  destination: string
+  newSize?: { width: number, height: number }
+  keepOriginal?: boolean // default false
+}) {
+  const { path, destination, newSize, keepOriginal = false } = options
+
+  const extension = getLowercaseExtension(path)
+
+  if (path === destination) {
+    throw new Error('Jimp/FFmpeg needs an input path different that the output path.')
+  }
+
+  logger.debug('Processing image %s to %s.', path, destination)
+
+  // Use FFmpeg to process GIF
+  if (extension === '.gif' || extension === '.webp') {
+    await processImageByFFmpeg({ path, destination, newSize })
+  } else {
+    await jimpProcessor({ path, destination, newSize, inputExt: extension })
+  }
+
+  if (keepOriginal !== true) await remove(path)
+
+  logger.debug('Finished processing image %s to %s.', path, destination)
+}
+
+export async function getImageSize (path: string) {
+  const inputBuffer = await readFile(path)
+
+  const Jimp = await import('jimp')
+
+  const image = await Jimp.default.read(inputBuffer)
+
+  return {
+    width: image.getWidth(),
+    height: image.getHeight()
+  }
+}
+
+// Build new size if height or width is missing, to keep the aspect ratio
+export async function buildImageSize (imagePath: string, sizeArg: { width?: number, height?: number }) {
+  if (sizeArg.width && sizeArg.height) {
+    return sizeArg as { width: number, height: number }
+  }
+
+  const size = await getImageSize(imagePath)
+  const ratio = size.width / size.height
+
+  return {
+    width: sizeArg.width ?? Math.round(sizeArg.height * ratio),
+    height: sizeArg.height ?? Math.round(sizeArg.width / ratio)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function jimpProcessor (options: {
+  path: string
+  destination: string
+  newSize?: {
+    width: number
+    height: number
+  }
+  inputExt: string
+}) {
+  const { path, newSize, inputExt, destination } = options
+
+  const inputBuffer = await readFile(path)
+
+  const Jimp = await import('jimp')
+  const sourceImage = await Jimp.default.read(inputBuffer)
+
+  await remove(destination)
+
+  // Optimization if the source file has the appropriate size
+
+  if (
+    skipProcessing({
+      sourceImage,
+      newSize,
+      imageBytes: inputBuffer.byteLength,
+      inputExt,
+      outputExt: getLowercaseExtension(destination)
+    })
+  ) {
+    return copy(path, destination)
+  }
+
+  if (newSize) {
+    await autoResize({ sourceImage, newSize, destination })
+  } else {
+    await write(sourceImage, destination)
+  }
+}
+
+function autoResize (options: {
+  sourceImage: Jimp
+  newSize: { width: number, height: number }
+  destination: string
+}) {
+  const { sourceImage, newSize, destination } = options
+
+  // Portrait mode targeting a landscape, apply some effect on the image
+  const sourceIsPortrait = sourceImage.getWidth() <= sourceImage.getHeight()
+  const destIsPortraitOrSquare = newSize.width <= newSize.height
+
+  removeExif(sourceImage)
+
+  if (sourceIsPortrait && !destIsPortraitOrSquare) {
+    const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height)
+      .color([ { apply: ColorActionName.SHADE, params: [ 50 ] } ])
+
+    const topImage = sourceImage.cloneQuiet().contain(newSize.width, newSize.height)
+
+    return write(baseImage.blit(topImage, 0, 0), destination)
+  }
+
+  return write(sourceImage.cover(newSize.width, newSize.height), destination)
+}
+
+function write (image: Jimp, destination: string) {
+  return image.quality(80).writeAsync(destination)
+}
+
+function skipProcessing (options: {
+  sourceImage: Jimp
+  newSize?: { width: number, height: number }
+  imageBytes: number
+  inputExt: string
+  outputExt: string
+}) {
+  const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options
+
+  if (hasExif(sourceImage)) return false
+  if (newSize && (sourceImage.getWidth() !== newSize.width || sourceImage.getHeight() !== newSize.height)) return false
+  if (inputExt !== outputExt) return false
+
+  const kB = 1000
+
+  if (newSize) {
+    if (newSize.height >= 1000) return imageBytes <= 200 * kB
+    if (newSize.height >= 500) return imageBytes <= 100 * kB
+  }
+
+  return imageBytes <= 15 * kB
+}
+
+function hasExif (image: Jimp) {
+  return !!(image.bitmap as any).exifBuffer
+}
+
+function removeExif (image: Jimp) {
+  ;(image.bitmap as any).exifBuffer = null
+}

server/core/helpers/logger.ts

@@ -0,0 +1,232 @@
+import { context, trace } from '@opentelemetry/api'
+import { omit } from '@peertube/peertube-core-utils'
+import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
+import { stat } from 'fs/promises'
+import { join } from 'path'
+import { format as sqlFormat } from 'sql-formatter'
+import { isatty } from 'tty'
+import { createLogger, format, transport, transports } from 'winston'
+import { FileTransportOptions } from 'winston/lib/winston/transports/index.js'
+import { isMainThread } from 'worker_threads'
+import { CONFIG } from '../initializers/config.js'
+import { LOG_FILENAME } from '../initializers/constants.js'
+
+const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
+
+const consoleLoggerFormat = format.printf(info => {
+  let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2)
+
+  if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = ''
+  else additionalInfos = ' ' + additionalInfos
+
+  if (info.sql) {
+    if (CONFIG.LOG.PRETTIFY_SQL) {
+      additionalInfos += '\n' + sqlFormat(info.sql as string, {
+        language: 'postgresql',
+        tabWidth: 2
+      })
+    } else {
+      additionalInfos += ' - ' + info.sql
+    }
+  }
+
+  return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message}${additionalInfos}`
+})
+
+export const jsonLoggerFormat: ReturnType<typeof format.printf> = format.printf(info => {
+  return JSON.stringify(info, removeCyclicValues())
+})
+
+export const labelFormatter: (suffix?: string) => ReturnType<typeof format.printf> = (suffix?: string) => {
+  return format.label({
+    label: suffix ? `${label} ${suffix}` : label
+  })
+}
+
+export function buildLogger (options: {
+  labelSuffix?: string
+  handleExceptions?: boolean // default false
+}) {
+  const { labelSuffix, handleExceptions = false } = options
+
+  const formatters = [
+    format.timestamp({
+      format: 'YYYY-MM-DD HH:mm:ss.SSS'
+    })
+  ]
+
+  if (doesConsoleSupportColor()) formatters.push(format.colorize())
+
+  formatters.push(consoleLoggerFormat)
+
+  const consoleTransport = new transports.Console({
+    handleExceptions,
+    format: format.combine(...formatters)
+  })
+
+  const fileLoggerOptions: FileTransportOptions = {
+    filename: join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME),
+    handleExceptions,
+    format: format.combine(
+      format.timestamp(),
+      jsonLoggerFormat
+    )
+  }
+
+  if (CONFIG.LOG.ROTATION.ENABLED) {
+    fileLoggerOptions.maxsize = CONFIG.LOG.ROTATION.MAX_FILE_SIZE
+    fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES
+  }
+
+  const loggerTransports: transport[] = []
+
+  // Don't add file logger transport in worker threads in production
+  // See https://github.com/winstonjs/winston/issues/2393
+  if (isMainThread || isTestOrDevInstance()) {
+    loggerTransports.push(new transports.File(fileLoggerOptions))
+  }
+
+  loggerTransports.push(consoleTransport)
+
+  return createLogger({
+    level: process.env.LOGGER_LEVEL ?? CONFIG.LOG.LEVEL,
+    defaultMeta: {
+      get traceId () {
+        return trace.getSpanContext(context.active())?.traceId
+      },
+      get spanId () {
+        return trace.getSpanContext(context.active())?.spanId
+      },
+      get traceFlags () {
+        return trace.getSpanContext(context.active())?.traceFlags
+      }
+    },
+    format: format.combine(
+      labelFormatter(labelSuffix),
+      format.splat()
+    ),
+    transports: loggerTransports,
+    exitOnError: true
+  })
+}
+
+export const logger = buildLogger({ handleExceptions: true })
+
+// ---------------------------------------------------------------------------
+// Bunyan logger adapter for Winston
+// ---------------------------------------------------------------------------
+
+export const bunyanLogger = {
+  level: () => {},
+  trace: bunyanLogFactory('debug'),
+  debug: bunyanLogFactory('debug'),
+  verbose: bunyanLogFactory('debug'),
+  info: bunyanLogFactory('info'),
+  warn: bunyanLogFactory('warn'),
+  error: bunyanLogFactory('error'),
+  fatal: bunyanLogFactory('error')
+}
+
+function bunyanLogFactory (level: string) {
+  return function (...params: any[]) {
+    let meta = null
+    let args = [].concat(params)
+
+    if (arguments[0] instanceof Error) {
+      meta = arguments[0].toString()
+      args = Array.prototype.slice.call(arguments, 1)
+      args.push(meta)
+    } else if (typeof (args[0]) !== 'string') {
+      meta = arguments[0]
+      args = Array.prototype.slice.call(arguments, 1)
+      args.push(meta)
+    }
+
+    logger[level].apply(logger, args)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Logger tags helpers
+// ---------------------------------------------------------------------------
+
+export type LoggerTags = { tags: (string | number)[] }
+export type LoggerTagsFn = (...tags: (string | number)[]) => LoggerTags
+export function loggerTagsFactory (...defaultTags: (string | number)[]): LoggerTagsFn {
+  return (...tags: (string | number)[]) => {
+    return { tags: defaultTags.concat(tags) }
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export async function mtimeSortFilesDesc (files: string[], basePath: string) {
+  const promises = []
+  const out: { file: string, mtime: number }[] = []
+
+  for (const file of files) {
+    const p = stat(basePath + '/' + file)
+      .then(stats => {
+        if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
+      })
+
+    promises.push(p)
+  }
+
+  await Promise.all(promises)
+
+  out.sort((a, b) => b.mtime - a.mtime)
+
+  return out
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function removeCyclicValues () {
+  const seen = new WeakSet()
+
+  // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples
+  return (key: string, value: any) => {
+    if (key === 'cert') return 'Replaced by the logger to avoid large log message'
+
+    if (typeof value === 'object' && value !== null) {
+      if (seen.has(value)) return
+
+      seen.add(value)
+    }
+
+    if (value instanceof Set) {
+      return Array.from(value)
+    }
+
+    if (value instanceof Map) {
+      return Array.from(value.entries())
+    }
+
+    if (value instanceof Error) {
+      const error = {}
+
+      Object.getOwnPropertyNames(value).forEach(key => {
+        error[key] = value[key]
+      })
+
+      return error
+    }
+
+    return value
+  }
+}
+
+function getAdditionalInfo (info: any) {
+  const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ]
+
+  return omit(info, toOmit)
+}
+
+function doesConsoleSupportColor () {
+  if (isTestOrDevInstance()) return true
+
+  return isatty(1) && process.env.TERM && process.env.TERM !== 'dumb'
+}

server/core/helpers/markdown.ts

@@ -0,0 +1,96 @@
+import {
+  getDefaultSanitizeOptions,
+  getMailHtmlSanitizeOptions,
+  getTextOnlySanitizeOptions,
+  TEXT_WITH_HTML_RULES
+} from '@peertube/peertube-core-utils'
+import MarkdownItClass from 'markdown-it'
+import markdownItEmoji from 'markdown-it-emoji/lib/light.mjs'
+import sanitizeHtml from 'sanitize-html'
+
+const defaultSanitizeOptions = getDefaultSanitizeOptions()
+const textOnlySanitizeOptions = getTextOnlySanitizeOptions()
+const hrefOnlySanitizeOptions = getMailHtmlSanitizeOptions()
+
+const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true })
+  .enable(TEXT_WITH_HTML_RULES)
+  .use(markdownItEmoji)
+
+const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false })
+  .use(markdownItEmoji)
+  .use(plainTextPlugin)
+
+export const toSafeHtml = (text: string) => {
+  if (!text) return ''
+
+  // Restore line feed
+  const textWithLineFeed = text.replace(/<br.?\/?>/g, '\r\n')
+
+  // Convert possible markdown (emojis, emphasis and lists) to html
+  const html = markdownItForSafeHtml.render(textWithLineFeed)
+
+  // Convert to safe Html
+  return sanitizeHtml(html, defaultSanitizeOptions)
+}
+
+export const mdToPlainText = (text: string) => {
+  if (!text) return ''
+
+  markdownItForPlainText.render(text)
+
+  // Convert to safe Html
+  return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions)
+}
+
+export const toSafeMailHtml = (text: string) => {
+  if (!text) return ''
+
+  return sanitizeHtml(text, hrefOnlySanitizeOptions)
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+// Thanks: https://github.com/wavesheep/markdown-it-plain-text
+function plainTextPlugin (markdownIt: any) {
+  function plainTextRule (state: any) {
+    const text = scan(state.tokens)
+
+    markdownIt.plainText = text
+  }
+
+  function scan (tokens: any[]) {
+    let lastSeparator = ''
+    let text = ''
+
+    function buildSeparator (token: any) {
+      if (token.type === 'list_item_close') {
+        lastSeparator = ', '
+      }
+
+      if (token.tag === 'br' || token.type === 'paragraph_close') {
+        lastSeparator = ' '
+      }
+    }
+
+    for (const token of tokens) {
+      buildSeparator(token)
+
+      if (token.type !== 'inline') continue
+
+      for (const child of token.children) {
+        buildSeparator(child)
+
+        if (!child.content) continue
+
+        text += lastSeparator + child.content
+        lastSeparator = ''
+      }
+    }
+
+    return text
+  }
+
+  markdownIt.core.ruler.push('plainText', plainTextRule)
+}

server/core/helpers/memoize.ts

@@ -0,0 +1,12 @@
+import memoizee from 'memoizee'
+
+export function Memoize (config?: memoizee.Options<any>) {
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const oldFunction = descriptor.value
+    const newFunction = memoizee(oldFunction, config)
+
+    descriptor.value = function () {
+      return newFunction.apply(this, arguments)
+    }
+  }
+}

server/core/helpers/mentions.ts

@@ -0,0 +1,40 @@
+import { uniqify } from '@peertube/peertube-core-utils'
+import { WEBSERVER } from '@server/initializers/constants.js'
+import { actorNameAlphabet } from './custom-validators/activitypub/actor.js'
+import { regexpCapture } from './regexp.js'
+
+export function extractMentions (text: string, isLocal: boolean) {
+  let result: string[] = []
+
+  const localMention = `@(${actorNameAlphabet}+)`
+  const remoteMention = `${localMention}@${WEBSERVER.HOST}`
+
+  const mentionRegex = isLocal
+    ? '(?:(?:' + remoteMention + ')|(?:' + localMention + '))' // Include local mentions?
+    : '(?:' + remoteMention + ')'
+
+  const firstMentionRegex = new RegExp(`^${mentionRegex} `, 'g')
+  const endMentionRegex = new RegExp(` ${mentionRegex}$`, 'g')
+  const remoteMentionsRegex = new RegExp(' ' + remoteMention + ' ', 'g')
+
+  result = result.concat(
+    regexpCapture(text, firstMentionRegex)
+      .map(([ , username1, username2 ]) => username1 || username2),
+    regexpCapture(text, endMentionRegex)
+      .map(([ , username1, username2 ]) => username1 || username2),
+    regexpCapture(text, remoteMentionsRegex)
+      .map(([ , username ]) => username)
+  )
+
+  // Include local mentions
+  if (isLocal) {
+    const localMentionsRegex = new RegExp(' ' + localMention + ' ', 'g')
+
+    result = result.concat(
+      regexpCapture(text, localMentionsRegex)
+        .map(([ , username ]) => username)
+    )
+  }
+
+  return uniqify(result)
+}

server/core/helpers/otp.ts

@@ -0,0 +1,64 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { WEBSERVER } from '@server/initializers/constants.js'
+import { Secret, TOTP } from 'otpauth'
+import { logger } from './logger.js'
+import { decrypt } from './peertube-crypto.js'
+
+async function isOTPValid (options: {
+  encryptedSecret: string
+  token: string
+}) {
+  const { token, encryptedSecret } = options
+
+  try {
+    const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE)
+
+    const totp = new TOTP({
+      ...baseOTPOptions(),
+
+      secret
+    })
+
+    const delta = totp.validate({
+      token,
+      window: 1
+    })
+
+    if (delta === null) return false
+
+    return true
+  } catch (err) {
+    logger.error('Cannot decrypt/validate OTP', { err })
+
+    return false
+  }
+}
+
+function generateOTPSecret (email: string) {
+  const totp = new TOTP({
+    ...baseOTPOptions(),
+
+    label: email,
+    secret: new Secret()
+  })
+
+  return {
+    secret: totp.secret.base32,
+    uri: totp.toString()
+  }
+}
+
+export {
+  generateOTPSecret, isOTPValid
+}
+
+// ---------------------------------------------------------------------------
+
+function baseOTPOptions () {
+  return {
+    issuer: WEBSERVER.HOST,
+    algorithm: 'SHA1',
+    digits: 6,
+    period: 30
+  }
+}

server/core/helpers/peertube-crypto.ts

@@ -0,0 +1,140 @@
+import { ParsedDraftSignature, parseRequestSignature, verifyDraftSignature } from '@misskey-dev/node-http-message-signatures'
+import { sha256 } from '@peertube/peertube-node-utils'
+import { createCipheriv, createDecipheriv } from 'crypto'
+import { Request } from 'express'
+import { BCRYPT_SALT_SIZE, ENCRYPTION, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants.js'
+import { MActor } from '../types/models/index.js'
+import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils.js'
+import { logger } from './logger.js'
+
+function createPrivateAndPublicKeys () {
+  logger.info('Generating a RSA key...')
+
+  return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE)
+}
+
+// ---------------------------------------------------------------------------
+// User password checks
+// ---------------------------------------------------------------------------
+
+async function comparePassword (plainPassword: string, hashPassword: string) {
+  if (!plainPassword) return false
+
+  if (Buffer.byteLength(plainPassword, 'utf8') > 72) {
+    throw new Error('Cannot compare more than 72 bytes with bcrypt')
+  }
+
+  const { compare } = await import('bcrypt')
+
+  return compare(plainPassword, hashPassword)
+}
+
+async function cryptPassword (password: string) {
+  const { genSalt, hash } = await import('bcrypt')
+
+  const salt = await genSalt(BCRYPT_SALT_SIZE)
+
+  return hash(password, salt)
+}
+
+// ---------------------------------------------------------------------------
+// HTTP Signature
+// ---------------------------------------------------------------------------
+
+function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean {
+  if (req.headers['signature'] && req.headers['digest']) {
+    return buildDigest(rawBody.toString()) === req.headers['digest']
+  }
+
+  return true
+}
+
+async function isHTTPSignatureVerified (httpSignatureParsed: ParsedDraftSignature, actor: MActor): Promise<boolean> {
+  const result = await verifyDraftSignature(
+    httpSignatureParsed.value,
+    actor.publicKey,
+    msg => logger.debug('Error in verify draft signature: ' + msg)
+  )
+
+  return result === true
+}
+
+function parseHTTPSignature (req: Request, clockSkewSeconds?: number): ParsedDraftSignature {
+  const requiredHeaders = req.method === 'POST'
+    ? [ '(request-target)', 'host', 'digest' ]
+    : [ '(request-target)', 'host' ]
+
+  const clockSkew = clockSkewSeconds
+    ? clockSkewSeconds * 1000
+    : undefined
+
+  const parsed = parseRequestSignature(req, {
+    requiredComponents: {
+      draft: requiredHeaders
+    },
+    clockSkew: {
+      delay: clockSkew
+    }
+  })
+
+  if (parsed.version !== 'draft') {
+    throw new Error(`Only draft version of HTTP signature is supported`)
+  }
+
+  const parsedHeaders = parsed.value.params.headers
+  if (!parsedHeaders.includes('date') && !parsedHeaders.includes('(created)')) {
+    throw new Error(`date or (created) must be included in signature`)
+  }
+
+  return parsed
+}
+
+// ---------------------------------------------------------------------------
+
+function buildDigest (body: any) {
+  const rawBody = typeof body === 'string' ? body : JSON.stringify(body)
+
+  return 'SHA-256=' + sha256(rawBody, 'base64')
+}
+
+// ---------------------------------------------------------------------------
+// Encryption
+// ---------------------------------------------------------------------------
+
+async function encrypt (str: string, secret: string) {
+  const iv = await randomBytesPromise(ENCRYPTION.IV)
+
+  const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
+  const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv)
+
+  let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':'
+  encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING)
+  encrypted += cipher.final(ENCRYPTION.ENCODING)
+
+  return encrypted
+}
+
+async function decrypt (encryptedArg: string, secret: string) {
+  const [ ivStr, encryptedStr ] = encryptedArg.split(':')
+
+  const iv = Buffer.from(ivStr, 'hex')
+  const key = await scryptPromise(secret, ENCRYPTION.SALT, 32)
+
+  const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv)
+
+  return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8')
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  buildDigest,
+  comparePassword,
+  createPrivateAndPublicKeys,
+  cryptPassword,
+  decrypt,
+  encrypt,
+  isHTTPSignatureDigestValid,
+  isHTTPSignatureVerified,
+  parseHTTPSignature
+}

server/core/helpers/peertube-jsonld.ts

@@ -0,0 +1,153 @@
+import { omit } from '@peertube/peertube-core-utils'
+import { sha256 } from '@peertube/peertube-node-utils'
+import { createSign, createVerify } from 'crypto'
+import cloneDeep from 'lodash-es/cloneDeep.js'
+import { MActor } from '../types/models/index.js'
+import { getAllContext } from './activity-pub-utils.js'
+import { jsonld } from './custom-jsonld-signature.js'
+import { isArray } from './custom-validators/misc.js'
+import { logger } from './logger.js'
+import { assertIsInWorkerThread } from './threads.js'
+
+type ExpressRequest = { body: any }
+
+export function compactJSONLDAndCheckSignature (fromActor: MActor, req: ExpressRequest): Promise<boolean> {
+  if (req.body.signature.type === 'RsaSignature2017') {
+    return compactJSONLDAndCheckRSA2017Signature(fromActor, req)
+  }
+
+  logger.warn('Unknown JSON LD signature %s.', req.body.signature.type, req.body)
+
+  return Promise.resolve(false)
+}
+
+// Backward compatibility with "other" implementations
+export async function compactJSONLDAndCheckRSA2017Signature (fromActor: MActor, req: ExpressRequest) {
+  const compacted = await jsonldCompact(omit(req.body, [ 'signature' ]))
+
+  fixCompacted(req.body, compacted)
+
+  req.body = { ...compacted, signature: req.body.signature }
+
+  if (compacted['@include']) {
+    logger.warn('JSON-LD @include is not supported')
+    return false
+  }
+
+  const [ documentHash, optionsHash ] = await Promise.all([
+    hashObject(compacted),
+    createSignatureHash(req.body.signature)
+  ])
+
+  const toVerify = optionsHash + documentHash
+
+  const verify = createVerify('RSA-SHA256')
+  verify.update(toVerify, 'utf8')
+
+  return verify.verify(fromActor.publicKey, req.body.signature.signatureValue, 'base64')
+}
+
+function fixCompacted (original: any, compacted: any) {
+  if (!original || !compacted) return
+
+  for (const [ k, v ] of Object.entries(original)) {
+    if (k === '@context' || k === 'signature') continue
+    if (v === undefined || v === null) continue
+
+    const cv = compacted[k]
+    if (cv === undefined || cv === null) continue
+
+    if (typeof v === 'string') {
+      if (v === 'https://www.w3.org/ns/activitystreams#Public' && cv === 'as:Public') {
+        compacted[k] = v
+      }
+    }
+
+    if (isArray(v) && !isArray(cv)) {
+      compacted[k] = [ cv ]
+
+      for (let i = 0; i < v.length; i++) {
+        if (v[i] === 'https://www.w3.org/ns/activitystreams#Public' && cv[i] === 'as:Public') {
+          compacted[k][i] = v[i]
+        }
+      }
+    }
+
+    if (typeof v === 'object') {
+      fixCompacted(original[k], compacted[k])
+    }
+  }
+}
+
+export async function signJsonLDObject<T> (options: {
+  byActor: { url: string, privateKey: string }
+  data: T
+  disableWorkerThreadAssertion?: boolean
+}) {
+  const { byActor, data, disableWorkerThreadAssertion = false } = options
+
+  if (!disableWorkerThreadAssertion) assertIsInWorkerThread()
+
+  const signature = {
+    type: 'RsaSignature2017',
+    creator: byActor.url,
+    created: new Date().toISOString()
+  }
+
+  const [ documentHash, optionsHash ] = await Promise.all([
+    createDocWithoutSignatureHash(data),
+    createSignatureHash(signature)
+  ])
+
+  const toSign = optionsHash + documentHash
+
+  const sign = createSign('RSA-SHA256')
+  sign.update(toSign, 'utf8')
+
+  const signatureValue = sign.sign(byActor.privateKey, 'base64')
+  Object.assign(signature, { signatureValue })
+
+  return Object.assign(data, { signature })
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+async function hashObject (obj: any): Promise<any> {
+  const res = await jsonldNormalize(obj)
+
+  return sha256(res)
+}
+
+function jsonldCompact (obj: any) {
+  return (jsonld as any).promises.compact(obj, getAllContext())
+}
+
+function jsonldNormalize (obj: any) {
+  return (jsonld as any).promises.normalize(obj, {
+    safe: true,
+    algorithm: 'URDNA2015',
+    format: 'application/n-quads'
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function createSignatureHash (signature: any) {
+  return hashObject({
+    '@context': [
+      'https://w3id.org/security/v1',
+      { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' }
+    ],
+
+    ...omit(signature, [ 'type', 'id', 'signatureValue' ])
+  })
+}
+
+function createDocWithoutSignatureHash (doc: any) {
+  const docWithoutSignature = cloneDeep(doc)
+  delete docWithoutSignature.signature
+
+  return hashObject(docWithoutSignature)
+}

server/core/helpers/promise-cache.ts

@@ -0,0 +1,39 @@
+export class CachePromiseFactory <A, R> {
+  private readonly running = new Map<string, Promise<R>>()
+
+  constructor (
+    private readonly fn: (arg: A) => Promise<R>,
+    private readonly keyBuilder: (arg: A) => string
+  ) {
+  }
+
+  run (arg: A) {
+    return this.runWithContext(null, arg)
+  }
+
+  runWithContext (ctx: any, arg: A) {
+    const key = this.keyBuilder(arg)
+
+    if (this.running.has(key)) return this.running.get(key)
+
+    const p = this.fn.apply(ctx || this, [ arg ])
+
+    this.running.set(key, p)
+
+    return p.finally(() => this.running.delete(key))
+  }
+}
+
+export function CachePromise (options: {
+  keyBuilder: (...args: any[]) => string
+}) {
+  return function (_target, _key, descriptor: PropertyDescriptor) {
+    const promiseCache = new CachePromiseFactory(descriptor.value, options.keyBuilder)
+
+    descriptor.value = function () {
+      if (arguments.length !== 1) throw new Error('Cache promise only support methods with 1 argument')
+
+      return promiseCache.runWithContext(this, arguments[0])
+    }
+  }
+}

server/core/helpers/proxy.ts

@@ -0,0 +1,9 @@
+export function getProxy () {
+  return process.env.HTTPS_PROXY ||
+         process.env.HTTP_PROXY ||
+         undefined
+}
+
+export function isProxyEnabled () {
+  return !!getProxy()
+}

server/core/helpers/query.ts

@@ -0,0 +1,84 @@
+import { pick } from '@peertube/peertube-core-utils'
+import {
+  VideoChannelsSearchQueryAfterSanitize,
+  VideoPlaylistsSearchQueryAfterSanitize,
+  VideosCommonQueryAfterSanitize,
+  VideosSearchQueryAfterSanitize
+} from '@peertube/peertube-models'
+
+function pickCommonVideoQuery (query: VideosCommonQueryAfterSanitize) {
+  return pick(query, [
+    'start',
+    'count',
+    'sort',
+    'nsfw',
+    'nsfwFlagsIncluded',
+    'nsfwFlagsExcluded',
+    'isLive',
+    'includeScheduledLive',
+    'categoryOneOf',
+    'licenceOneOf',
+    'languageOneOf',
+    'host',
+    'privacyOneOf',
+    'tagsOneOf',
+    'tagsAllOf',
+    'isLocal',
+    'include',
+    'skipCount',
+    'hasHLSFiles',
+    'hasWebVideoFiles',
+    'search',
+    'excludeAlreadyWatched',
+    'autoTagOneOf'
+  ])
+}
+
+function pickSearchVideoQuery (query: VideosSearchQueryAfterSanitize) {
+  return {
+    ...pickCommonVideoQuery(query),
+
+    ...pick(query, [
+      'searchTarget',
+      'host',
+      'startDate',
+      'endDate',
+      'originallyPublishedStartDate',
+      'originallyPublishedEndDate',
+      'durationMin',
+      'durationMax',
+      'uuids'
+    ])
+  }
+}
+
+function pickSearchChannelQuery (query: VideoChannelsSearchQueryAfterSanitize) {
+  return pick(query, [
+    'searchTarget',
+    'search',
+    'start',
+    'count',
+    'sort',
+    'host',
+    'handles'
+  ])
+}
+
+function pickSearchPlaylistQuery (query: VideoPlaylistsSearchQueryAfterSanitize) {
+  return pick(query, [
+    'searchTarget',
+    'search',
+    'start',
+    'count',
+    'sort',
+    'host',
+    'uuids'
+  ])
+}
+
+export {
+  pickCommonVideoQuery,
+  pickSearchVideoQuery,
+  pickSearchPlaylistQuery,
+  pickSearchChannelQuery
+}

server/core/helpers/regexp.ts

@@ -0,0 +1,32 @@
+// Thanks to https://regex101.com
+export function regexpCapture (str: string, regex: RegExp, maxIterations = 100) {
+  const result: RegExpExecArray[] = []
+  let m: RegExpExecArray
+  let i = 0
+
+  while ((m = regex.exec(str)) !== null && i < maxIterations) {
+    // This is necessary to avoid infinite loops with zero-width matches
+    if (m.index === regex.lastIndex) {
+      regex.lastIndex++
+    }
+
+    result.push(m)
+    i++
+  }
+
+  return result
+}
+
+export function wordsToRegExp (words: string[]) {
+  if (words.length === 0) throw new Error('Need words with at least one element')
+
+  const innerRegex = words
+    .map(word => escapeForRegex(word.trim()))
+    .join('|')
+
+  return new RegExp(`(?:\\P{L}|^)(?:${innerRegex})(?=\\P{L}|$)`, 'iu')
+}
+
+export function escapeForRegex (value: string) {
+  return value.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')
+}

server/core/helpers/requests.ts

@@ -0,0 +1,289 @@
+import { signAsDraftToRequest } from '@misskey-dev/node-http-message-signatures'
+import { CONFIG } from '@server/initializers/config.js'
+import { createWriteStream } from 'fs'
+import { remove } from 'fs-extra/esm'
+import got, {
+  CancelableRequest,
+  OptionsInit,
+  OptionsOfTextResponseBody,
+  OptionsOfUnknownResponseBodyWrapped,
+  RequestError,
+  Response
+} from 'got'
+import { gotSsrf } from 'got-ssrf'
+import http from 'http'
+import https from 'https'
+import { HttpProxyAgent, HttpsProxyAgent } from '../helpers/hpagent.js'
+import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants.js'
+import { pipelinePromise } from './core-utils.js'
+import { logger, loggerTagsFactory } from './logger.js'
+import { getProxy, isProxyEnabled } from './proxy.js'
+
+const lTags = loggerTagsFactory('request')
+
+export interface PeerTubeRequestError extends Error {
+  statusCode?: number
+
+  responseBody?: any
+  responseHeaders?: any
+
+  requestHeaders?: any
+  requestUrl?: any
+  requestMethod?: any
+}
+
+export type PeerTubeRequestOptions = {
+  timeout?: number
+  activityPub?: boolean
+  bodyKBLimit?: number // 1MB
+
+  httpSignature?: {
+    keyId: string
+    key: string
+    headers: string[]
+  }
+
+  jsonResponse?: boolean
+
+  followRedirect?: boolean
+} & Pick<OptionsInit, 'headers' | 'json' | 'method' | 'searchParams'>
+
+export const unsafeSSRFGot = got.extend({
+  ...getProxyAgent(),
+
+  headers: {
+    'user-agent': getUserAgent()
+  },
+
+  handlers: [
+    (options, next) => {
+      const promiseOrStream = next(options) as CancelableRequest<any>
+      const bodyKBLimit = options.context?.bodyKBLimit as number
+      if (!bodyKBLimit) throw new Error('No KB limit for this request')
+
+      const bodyLimit = bodyKBLimit * 1000
+
+      void promiseOrStream.on('downloadProgress', progress => {
+        if (progress.transferred > bodyLimit && progress.percent !== 1) {
+          const message = `Exceeded the download limit of ${bodyLimit} B`
+          logger.warn(message, lTags())
+
+          // CancelableRequest
+          if (promiseOrStream.cancel) {
+            promiseOrStream.cancel()
+            return
+          }
+
+          ;(promiseOrStream as any).destroy()
+        }
+      })
+
+      return promiseOrStream
+    }
+  ],
+
+  hooks: {
+    beforeRequest: [
+      options => {
+        const headers = options.headers || {}
+        headers['host'] = buildUrl(options.url).host
+      },
+
+      async options => {
+        const httpSignatureOptions = options.context?.httpSignature as PeerTubeRequestOptions['httpSignature']
+
+        if (httpSignatureOptions) {
+          const method = options.method ?? 'GET'
+          const path = buildUrl(options.url).pathname
+
+          if (!method || !path) {
+            throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
+          }
+
+          const request = {
+            headers: options.headers,
+            method,
+            url: path
+          }
+
+          await signAsDraftToRequest(
+            request,
+            {
+              keyId: httpSignatureOptions.keyId,
+              privateKeyPem: httpSignatureOptions.key
+            },
+            httpSignatureOptions.headers
+          )
+        }
+      }
+    ],
+
+    beforeRetry: [
+      (error: RequestError, retryCount: number) => {
+        logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
+      }
+    ]
+  }
+})
+
+export const peertubeGot = CONFIG.FEDERATION.PREVENT_SSRF
+  ? got.extend(gotSsrf, unsafeSSRFGot)
+  : unsafeSSRFGot
+
+// ---------------------------------------------------------------------------
+
+export function doRequest (url: string, options: PeerTubeRequestOptions & { preventSSRF?: false } = {}) {
+  const gotOptions = buildGotOptions(options) as OptionsOfTextResponseBody
+
+  const gotInstance = options.preventSSRF === false
+    ? unsafeSSRFGot
+    : peertubeGot
+
+  return gotInstance(url, gotOptions)
+    .catch(err => {
+      throw buildRequestError(err)
+    })
+}
+
+export function doJSONRequest<T> (url: string, options: PeerTubeRequestOptions & { preventSSRF?: false } = {}) {
+  const gotOptions = buildGotOptions(options)
+
+  const gotInstance = options.preventSSRF === false
+    ? unsafeSSRFGot
+    : peertubeGot
+
+  return gotInstance<T>(url, { ...gotOptions, responseType: 'json' })
+    .catch(err => {
+      throw buildRequestError(err)
+    })
+}
+
+export async function doRequestAndSaveToFile (url: string, destPath: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
+
+  const outFile = createWriteStream(destPath)
+
+  try {
+    await pipelinePromise(
+      peertubeGot.stream(url, { ...gotOptions, isStream: true }),
+      outFile
+    )
+  } catch (err) {
+    remove(destPath)
+      .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
+
+    throw buildRequestError(err)
+  }
+}
+
+export function generateRequestStream (url: string, options: PeerTubeRequestOptions = {}) {
+  const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT })
+
+  return peertubeGot.stream(url, { ...gotOptions, isStream: true })
+}
+
+export function getProxyAgent () {
+  if (!isProxyEnabled()) {
+    return {
+      agent: { // Fix issue https://github.com/node-fetch/node-fetch/issues/1735 with Node 20
+        http: new http.Agent({ keepAlive: false }),
+        https: new https.Agent({ keepAlive: false })
+      }
+    }
+  }
+
+  const proxy = getProxy()
+
+  logger.info('Using proxy %s.', proxy, lTags())
+
+  const proxyAgentOptions = {
+    keepAlive: true,
+    keepAliveMsecs: 1000,
+    maxSockets: 256,
+    maxFreeSockets: 256,
+    scheduling: 'lifo' as 'lifo',
+    proxy
+  }
+
+  return {
+    agent: {
+      http: new HttpProxyAgent(proxyAgentOptions),
+      https: new HttpsProxyAgent(proxyAgentOptions)
+    }
+  }
+}
+
+export function isBinaryResponse (result: Response<any>) {
+  return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
+}
+
+export function buildRequestError (error: RequestError) {
+  if (!error.response && !error.options) return error
+
+  const newError: PeerTubeRequestError = new Error(error.message)
+  newError.name = error.name
+  newError.stack = error.stack
+
+  if (error.response) {
+    newError.responseBody = error.response.body
+    newError.responseHeaders = error.response.headers
+    newError.statusCode = error.response.statusCode
+  }
+
+  if (error.options) {
+    newError.requestHeaders = error.options.headers
+    newError.requestUrl = error.options.url
+    newError.requestMethod = error.options.method
+  }
+
+  return newError
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function getUserAgent () {
+  return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
+}
+
+function buildGotOptions (options: PeerTubeRequestOptions): OptionsOfUnknownResponseBodyWrapped {
+  const { activityPub, bodyKBLimit = 3000 } = options
+
+  const context = { bodyKBLimit, httpSignature: options.httpSignature }
+
+  let headers = options.headers || {}
+
+  if (!headers.date) {
+    headers = { ...headers, date: new Date().toUTCString() }
+  }
+
+  if (activityPub && !headers.accept) {
+    headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
+  }
+
+  return {
+    method: options.method,
+    dnsCache: true,
+    resolveBodyOnly: false,
+    timeout: {
+      request: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT
+    },
+    json: options.json,
+    searchParams: options.searchParams,
+    followRedirect: options.followRedirect,
+    retry: {
+      limit: 2
+    },
+    headers,
+    context
+  }
+}
+
+function buildUrl (url: string | URL) {
+  if (typeof url === 'string') {
+    return new URL(url)
+  }
+
+  return url
+}

server/core/helpers/stream-replacer.ts

@@ -0,0 +1,58 @@
+import { Transform, TransformCallback } from 'stream'
+
+// Thanks: https://stackoverflow.com/a/45126242
+class StreamReplacer extends Transform {
+  private pendingChunk: Buffer
+
+  constructor (private readonly replacer: (line: string) => string) {
+    super()
+  }
+
+  _transform (chunk: Buffer, _encoding: BufferEncoding, done: TransformCallback) {
+    try {
+      this.pendingChunk = this.pendingChunk?.length
+        ? Buffer.concat([ this.pendingChunk, chunk ])
+        : chunk
+
+      let index: number
+
+      // As long as we keep finding newlines, keep making slices of the buffer and push them to the
+      // readable side of the transform stream
+      while ((index = this.pendingChunk.indexOf('\n')) !== -1) {
+        // The `end` parameter is non-inclusive, so increase it to include the newline we found
+        const line = this.pendingChunk.subarray(0, ++index)
+
+        // `start` is inclusive, but we are already one char ahead of the newline -> all good
+        this.pendingChunk = this.pendingChunk.subarray(index)
+
+        // We have a single line here! Prepend the string we want
+        this.push(this.doReplace(line))
+      }
+
+      return done()
+    } catch (err) {
+      return done(err)
+    }
+  }
+
+  _flush (done: TransformCallback) {
+    // If we have any remaining data in the cache, send it out
+    if (!this.pendingChunk?.length) return done()
+
+    try {
+      return done(null, this.doReplace(this.pendingChunk))
+    } catch (err) {
+      return done(err)
+    }
+  }
+
+  private doReplace (buffer: Buffer) {
+    const line = this.replacer(buffer.toString('utf8'))
+
+    return Buffer.from(line, 'utf8')
+  }
+}
+
+export {
+  StreamReplacer
+}

server/core/helpers/threads.ts

@@ -0,0 +1,11 @@
+import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
+import { isMainThread } from 'node:worker_threads'
+import { logger } from './logger.js'
+
+export function assertIsInWorkerThread () {
+  if (!isMainThread) return
+
+  logger.error('Caller is not in worker thread', { stack: new Error().stack })
+
+  if (isTestOrDevInstance()) process.exit(1)
+}

server/core/helpers/token-generator.ts

@@ -0,0 +1,19 @@
+import { buildUUID } from '@peertube/peertube-node-utils'
+
+function generateRunnerRegistrationToken () {
+  return 'ptrrt-' + buildUUID()
+}
+
+function generateRunnerToken () {
+  return 'ptrt-' + buildUUID()
+}
+
+function generateRunnerJobToken () {
+  return 'ptrjt-' + buildUUID()
+}
+
+export {
+  generateRunnerRegistrationToken,
+  generateRunnerToken,
+  generateRunnerJobToken
+}

server/core/helpers/unzip.ts

@@ -0,0 +1,80 @@
+import { createWriteStream } from 'fs'
+import { ensureDir } from 'fs-extra/esm'
+import { dirname, join } from 'path'
+import { pipeline } from 'stream'
+import * as yauzl from 'yauzl'
+import { logger, loggerTagsFactory } from './logger.js'
+
+const lTags = loggerTagsFactory('unzip')
+
+export async function unzip (options: {
+  source: string
+  destination: string
+  maxSize: number // in bytes
+  maxFiles: number
+}) {
+  const { source, destination } = options
+
+  await ensureDir(destination)
+
+  logger.info(`Unzip ${source} to ${destination}`, lTags())
+
+  return new Promise<void>((res, rej) => {
+    yauzl.open(source, { lazyEntries: true }, (err, zipFile) => {
+      if (err) return rej(err)
+
+      zipFile.on('error', err => rej(err))
+
+      let decompressedSize = 0
+      let entries = 0
+
+      zipFile.readEntry()
+
+      zipFile.on('entry', async entry => {
+        decompressedSize += entry.uncompressedSize
+        entries++
+
+        if (decompressedSize > options.maxSize) {
+          zipFile.close()
+          return rej(new Error(`Unzipped size exceeds ${options.maxSize} bytes`))
+        }
+
+        if (entries > options.maxFiles) {
+          zipFile.close()
+          return rej(new Error(`Unzipped files count exceeds ${options.maxFiles}`))
+        }
+
+        const entryPath = join(destination, entry.fileName)
+
+        try {
+          if (entry.fileName.endsWith('/')) {
+            await ensureDir(entryPath)
+            logger.debug(`Creating directory from zip ${entryPath}`, lTags())
+
+            zipFile.readEntry()
+            return
+          }
+
+          await ensureDir(dirname(entryPath))
+        } catch (err) {
+          return rej(err)
+        }
+
+        zipFile.openReadStream(entry, (readErr, readStream) => {
+          if (readErr) return rej(readErr)
+
+          logger.debug(`Creating file from zip ${entryPath}`, lTags())
+
+          const writeStream = createWriteStream(entryPath)
+          writeStream.on('close', () => zipFile.readEntry())
+
+          pipeline(readStream, writeStream, pipelineErr => {
+            if (pipelineErr) return rej(pipelineErr)
+          })
+        })
+      })
+
+      zipFile.on('end', () => res())
+    })
+  })
+}

server/core/helpers/upload.ts

@@ -0,0 +1,14 @@
+import { join } from 'path'
+import { DIRECTORIES } from '@server/initializers/constants.js'
+
+function getResumableUploadPath (filename?: string) {
+  if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename)
+
+  return DIRECTORIES.RESUMABLE_UPLOAD
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getResumableUploadPath
+}

server/core/helpers/utils.ts

@@ -0,0 +1,70 @@
+import { remove } from 'fs-extra/esm'
+import { Instance as ParseTorrent } from 'parse-torrent'
+import { join } from 'path'
+import { sha256 } from '@peertube/peertube-node-utils'
+import { ResultList } from '@peertube/peertube-models'
+import { CONFIG } from '../initializers/config.js'
+import { randomBytesPromise } from './core-utils.js'
+import { logger } from './logger.js'
+
+function deleteFileAndCatch (path: string) {
+  remove(path)
+    .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
+}
+
+async function generateRandomString (size: number) {
+  const raw = await randomBytesPromise(size)
+
+  return raw.toString('hex')
+}
+
+interface FormattableToJSON<U, V> {
+  toFormattedJSON (args?: U): V
+}
+
+function getFormattedObjects<U, V, T extends FormattableToJSON<U, V>> (objects: T[], objectsTotal: number, formattedArg?: U) {
+  const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg))
+
+  return {
+    total: objectsTotal,
+    data: formattedObjects
+  } as ResultList<V>
+}
+
+function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') {
+  const id = typeof target === 'string'
+    ? target
+    : target.infoHash
+
+  const hash = sha256(id)
+  return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`)
+}
+
+function getSecureTorrentName (originalName: string) {
+  return sha256(originalName) + '.torrent'
+}
+
+/**
+ * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns
+ * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does
+ * not contain a UUID, returns null.
+ */
+function getUUIDFromFilename (filename: string) {
+  const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/
+  const result = filename.match(regex)
+
+  if (!result || Array.isArray(result) === false) return null
+
+  return result[0]
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  deleteFileAndCatch,
+  generateRandomString,
+  getFormattedObjects,
+  getSecureTorrentName,
+  generateVideoImportTmpPath,
+  getUUIDFromFilename
+}

server/core/helpers/version.ts

@@ -0,0 +1,29 @@
+import { execPromise, execPromise2 } from './core-utils.js'
+import { logger } from './logger.js'
+
+export async function getServerCommit () {
+  try {
+    const tag = await execPromise2(
+      '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true',
+      { stdio: [ 0, 1, 2 ] }
+    )
+
+    if (tag) return tag.replace(/^v/, '')
+  } catch (err) {
+    logger.debug('Cannot get version from git tags.', { err })
+  }
+
+  try {
+    const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD')
+
+    if (version) return version.toString().trim()
+  } catch (err) {
+    logger.debug('Cannot get version from git HEAD.', { err })
+  }
+
+  return ''
+}
+
+export function getNodeABIVersion () {
+  return process.versions.modules
+}

server/core/helpers/video.ts

@@ -0,0 +1,52 @@
+import { VideoPrivacy } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js'
+import { Response } from 'express'
+
+export function getVideoWithAttributes (res: Response) {
+  return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo
+}
+
+export function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
+  return isStreamingPlaylist(videoOrPlaylist)
+    ? videoOrPlaylist.Video
+    : videoOrPlaylist
+}
+
+export function getPrivaciesForFederation () {
+  return (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true)
+    ? [ { privacy: VideoPrivacy.PUBLIC }, { privacy: VideoPrivacy.UNLISTED } ]
+    : [ { privacy: VideoPrivacy.PUBLIC } ]
+}
+
+export function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) {
+  const value = mimeTypes[mimeType]
+
+  if (Array.isArray(value)) return value[0]
+
+  return value
+}
+
+export function peertubeLicenceToSPDX (licence: number) {
+  return {
+    1: 'CC-BY-4.0',
+    2: 'CC-BY-SA-4.0',
+    3: 'CC-BY-ND-4.0',
+    4: 'CC-BY-NC-4.0',
+    5: 'CC-BY-NC-SA-4.0',
+    6: 'CC-BY-NC-ND-4.0',
+    7: 'CC0'
+  }[licence]
+}
+
+export function spdxToPeertubeLicence (licence: string) {
+  return {
+    'CC-BY-4.0': 1,
+    'CC-BY-SA-4.0': 2,
+    'CC-BY-ND-4.0': 3,
+    'CC-BY-NC-4.0': 4,
+    'CC-BY-NC-SA-4.0': 5,
+    'CC-BY-NC-ND-4.0': 6,
+    'CC0': 7
+  }[licence]
+}

server/core/helpers/youtube-dl/index.ts

@@ -0,0 +1,3 @@
+export * from './youtube-dl-cli.js'
+export * from './youtube-dl-info-builder.js'
+export * from './youtube-dl-wrapper.js'

server/core/helpers/youtube-dl/youtube-dl-cli.ts

@@ -0,0 +1,284 @@
+import { randomInt } from '@peertube/peertube-core-utils'
+import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { execa, Options as ExecaNodeOptions } from 'execa'
+import { ensureDir, pathExists } from 'fs-extra/esm'
+import { chmod, writeFile } from 'fs/promises'
+import { OptionsOfBufferResponseBody } from 'got'
+import { dirname, join } from 'path'
+import { logger, loggerTagsFactory } from '../logger.js'
+import { getProxy, isProxyEnabled } from '../proxy.js'
+import { isBinaryResponse, unsafeSSRFGot } from '../requests.js'
+
+type ProcessOptions = Pick<ExecaNodeOptions, 'cwd' | 'maxBuffer'>
+
+const lTags = loggerTagsFactory('youtube-dl')
+
+const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME)
+
+export class YoutubeDLCLI {
+  static async safeGet () {
+    if (!await pathExists(youtubeDLBinaryPath)) {
+      await ensureDir(dirname(youtubeDLBinaryPath))
+
+      await this.updateYoutubeDLBinary()
+    }
+
+    return new YoutubeDLCLI()
+  }
+
+  static async updateYoutubeDLBinary () {
+    const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL
+
+    logger.info('Updating youtubeDL binary from %s.', url, lTags())
+
+    const gotOptions: OptionsOfBufferResponseBody = {
+      context: { bodyKBLimit: 100_000 },
+      responseType: 'buffer' as 'buffer'
+    }
+
+    if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) {
+      gotOptions.headers = {
+        authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN
+      }
+    }
+
+    try {
+      let gotResult = await unsafeSSRFGot(url, gotOptions)
+
+      if (!isBinaryResponse(gotResult)) {
+        const json = JSON.parse(gotResult.body.toString())
+        const latest = json.filter(release => release.prerelease === false)[0]
+        if (!latest) throw new Error('Cannot find latest release')
+
+        const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME
+        const releaseAsset = latest.assets.find(a => a.name === releaseName)
+        if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`)
+
+        gotResult = await unsafeSSRFGot(releaseAsset.browser_download_url, gotOptions)
+      }
+
+      if (!isBinaryResponse(gotResult)) {
+        throw new Error('Not a binary response')
+      }
+
+      await writeFile(youtubeDLBinaryPath, gotResult.body)
+
+      if (!CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.PYTHON_PATH) {
+        await chmod(youtubeDLBinaryPath, '744')
+      }
+
+      logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags())
+    } catch (err) {
+      logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() })
+    }
+  }
+
+  static getYoutubeDLVideoFormat (enabledResolutions: VideoResolutionType[], useBestFormat: boolean) {
+    /**
+     * list of format selectors in order or preference
+     * see https://github.com/ytdl-org/youtube-dl#format-selection
+     *
+     * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope
+     * of being able to do a "quick-transcode"
+     * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9)
+     * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback
+     *
+     * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499
+     */
+
+    let result: string[] = []
+
+    if (!useBestFormat) {
+      const resolution = enabledResolutions.length === 0
+        ? VideoResolution.H_720P
+        : Math.max(...enabledResolutions)
+
+      result = [
+        `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1
+        `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2
+        `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case #
+      ]
+    }
+
+    return result.concat([
+      'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio',
+      'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats
+      'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
+      'best' // Ultimate fallback
+    ]).join('/')
+  }
+
+  private constructor () {
+  }
+
+  download (options: {
+    url: string
+    format: string
+    output: string
+    processOptions: ProcessOptions
+    timeout?: number
+    additionalYoutubeDLArgs?: string[]
+  }) {
+    let args = options.additionalYoutubeDLArgs || []
+    args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ])
+
+    return this.run({
+      url: options.url,
+      processOptions: options.processOptions,
+      timeout: options.timeout,
+      args
+    })
+  }
+
+  async getInfo (options: {
+    url: string
+    format: string
+    processOptions: ProcessOptions
+    additionalYoutubeDLArgs?: string[]
+  }) {
+    const { url, format, additionalYoutubeDLArgs = [], processOptions } = options
+
+    const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ])
+
+    const data = await this.run({ url, args: completeArgs, processOptions })
+    if (!data) return undefined
+
+    const info = data.map(d => JSON.parse(d))
+
+    return info.length === 1
+      ? info[0]
+      : info
+  }
+
+  async getListInfo (options: {
+    url: string
+    latestVideosCount?: number
+    processOptions: ProcessOptions
+  }): Promise<{ upload_date: string, webpage_url: string }[]> {
+    const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ]
+
+    if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
+      // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel
+      additionalYoutubeDLArgs.push('--flat-playlist')
+    }
+
+    if (options.latestVideosCount !== undefined) {
+      additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString())
+    }
+
+    const result = await this.getInfo({
+      url: options.url,
+      format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false),
+      processOptions: options.processOptions,
+      additionalYoutubeDLArgs
+    })
+
+    if (!result) return result
+    if (!Array.isArray(result)) return [ result ]
+
+    return result
+  }
+
+  async getSubs (options: {
+    url: string
+    format: 'vtt'
+    processOptions: ProcessOptions
+  }) {
+    const { url, format, processOptions } = options
+
+    const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ]
+
+    const data = await this.run({ url, args, processOptions })
+    const files: string[] = []
+
+    const skipString = '[info] Writing video subtitles to: '
+
+    for (let i = 0, len = data.length; i < len; i++) {
+      const line = data[i]
+
+      if (line.startsWith(skipString)) {
+        files.push(line.slice(skipString.length))
+      }
+    }
+
+    return files
+  }
+
+  private async run (options: {
+    url: string
+    args: string[]
+    timeout?: number
+    processOptions: ProcessOptions
+  }) {
+    const { url, args, timeout, processOptions } = options
+
+    let completeArgs = this.wrapWithJSRuntimeOptions(args)
+    completeArgs = this.wrapWithProxyOptions(completeArgs)
+    completeArgs = this.wrapWithIPOptions(completeArgs)
+    completeArgs = this.wrapWithFFmpegOptions(completeArgs)
+
+    const subProcessBinary = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.PYTHON_PATH || youtubeDLBinaryPath
+    const subProcessArgs = [ ...completeArgs, url ]
+    if (subProcessBinary !== youtubeDLBinaryPath) subProcessArgs.unshift(youtubeDLBinaryPath)
+
+    const subProcess = execa(subProcessBinary, subProcessArgs, processOptions)
+
+    if (timeout) {
+      setTimeout(() => subProcess.kill(), timeout)
+    }
+
+    const output = await subProcess
+
+    logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() })
+
+    return output.stdout
+      ? output.stdout.trim().split(/\r?\n/)
+      : undefined
+  }
+
+  private wrapWithJSRuntimeOptions (args: string[]) {
+    if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') {
+      return [ '--js-runtimes', 'node' ].concat(args)
+    }
+
+    return args
+  }
+
+  private wrapWithProxyOptions (args: string[]) {
+    const config = CONFIG.IMPORT.VIDEOS.HTTP.PROXIES
+    const configProxyEnabled = Array.isArray(config) && config.length !== 0
+
+    if (configProxyEnabled || isProxyEnabled()) {
+      const proxy = configProxyEnabled
+        ? config[randomInt(0, config.length)]
+        : getProxy()
+
+      logger.debug('Using proxy %s for YoutubeDL', proxy, lTags())
+
+      return [ '--proxy', proxy ].concat(args)
+    }
+
+    return args
+  }
+
+  private wrapWithIPOptions (args: string[]) {
+    if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) {
+      logger.debug('Force ipv4 for YoutubeDL')
+
+      return [ '--force-ipv4' ].concat(args)
+    }
+
+    return args
+  }
+
+  private wrapWithFFmpegOptions (args: string[]) {
+    if (process.env.FFMPEG_PATH) {
+      logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags())
+
+      return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args)
+    }
+
+    return args
+  }
+}

server/core/helpers/youtube-dl/youtube-dl-info-builder.ts

@@ -0,0 +1,213 @@
+import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants.js'
+import { peertubeTruncate } from '../core-utils.js'
+import { isUrlValid } from '../custom-validators/activitypub/misc.js'
+import { isArray } from '../custom-validators/misc.js'
+
+export type YoutubeDLInfo = {
+  name?: string
+  description?: string
+  category?: number
+  language?: string
+  licence?: number
+  nsfw?: boolean
+  tags?: string[]
+  thumbnailUrl?: string
+  ext?: string
+  originallyPublishedAtWithoutTime?: Date
+  webpageUrl?: string
+
+  urls?: string[]
+
+  chapters?: {
+    timecode: number
+    title: string
+  }[]
+}
+
+export class YoutubeDLInfoBuilder {
+  private readonly info: any
+
+  constructor (info: any) {
+    this.info = { ...info }
+  }
+
+  getInfo () {
+    const obj = this.buildVideoInfo(this.normalizeObject(this.info))
+    if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video'
+
+    return obj
+  }
+
+  private normalizeObject (obj: any) {
+    const newObj: any = {}
+
+    for (const key of Object.keys(obj)) {
+      // Deprecated key
+      if (key === 'resolution') continue
+
+      const value = obj[key]
+
+      if (typeof value === 'string') {
+        newObj[key] = value.normalize()
+      } else {
+        newObj[key] = value
+      }
+    }
+
+    return newObj
+  }
+
+  private buildOriginallyPublishedAt (obj: any) {
+    let originallyPublishedAt: Date = null
+
+    const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date)
+    if (uploadDateMatcher) {
+      originallyPublishedAt = new Date()
+      originallyPublishedAt.setHours(0, 0, 0, 0)
+
+      const year = parseInt(uploadDateMatcher[1], 10)
+      // Month starts from 0
+      const month = parseInt(uploadDateMatcher[2], 10) - 1
+      const day = parseInt(uploadDateMatcher[3], 10)
+
+      originallyPublishedAt.setFullYear(year, month, day)
+    }
+
+    return originallyPublishedAt
+  }
+
+  private buildVideoInfo (obj: any): YoutubeDLInfo {
+    return {
+      name: this.titleTruncation(obj.title || obj.fulltitle),
+      description: this.descriptionTruncation(obj.description),
+      category: this.getCategory(obj.categories),
+      licence: this.getLicence(obj.license),
+      language: this.getLanguage(obj.language),
+      nsfw: this.isNSFW(obj),
+      tags: this.getTags(obj.tags),
+      thumbnailUrl: obj.thumbnail || undefined,
+      urls: this.buildAvailableUrl(obj),
+      originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj),
+      ext: obj.ext,
+      webpageUrl: obj.webpage_url,
+
+      chapters: isArray(obj.chapters)
+        ? obj.chapters.map((c: { start_time: number, title: string }) => {
+          return {
+            timecode: c.start_time,
+            title: c.title.slice(0, CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE.max)
+          }
+        })
+        : []
+    }
+  }
+
+  private buildAvailableUrl (obj: any) {
+    const urls: string[] = []
+
+    if (obj.url) urls.push(obj.url)
+    if (obj.urls) {
+      if (Array.isArray(obj.urls)) urls.push(...obj.urls)
+      else urls.push(obj.urls)
+    }
+
+    const formats = Array.isArray(obj.formats)
+      ? obj.formats
+      : []
+
+    for (const format of formats) {
+      if (!format.url) continue
+
+      urls.push(format.url)
+    }
+
+    const thumbnails = Array.isArray(obj.thumbnails)
+      ? obj.thumbnails
+      : []
+
+    for (const thumbnail of thumbnails) {
+      if (!thumbnail.url) continue
+
+      urls.push(thumbnail.url)
+    }
+
+    if (obj.thumbnail) urls.push(obj.thumbnail)
+
+    for (const subtitleKey of Object.keys(obj.subtitles || {})) {
+      const subtitles = obj.subtitles[subtitleKey]
+      if (!Array.isArray(subtitles)) continue
+
+      for (const subtitle of subtitles) {
+        if (!subtitle.url) continue
+
+        urls.push(subtitle.url)
+      }
+    }
+
+    return urls.filter(u => u && isUrlValid(u))
+  }
+
+  private titleTruncation (title: string) {
+    return peertubeTruncate(title, {
+      length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max,
+      separator: /,? +/,
+      omission: ' […]'
+    })
+  }
+
+  private descriptionTruncation (description: string) {
+    if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined
+
+    return peertubeTruncate(description, {
+      length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max,
+      separator: /,? +/,
+      omission: ' […]'
+    })
+  }
+
+  private isNSFW (info: any) {
+    return info?.age_limit >= 16
+  }
+
+  private getTags (tags: string[]) {
+    if (Array.isArray(tags) === false) return []
+
+    return tags
+      .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min)
+      .map(t => t.normalize())
+      .slice(0, 5)
+  }
+
+  private getLicence (licence: string) {
+    if (!licence) return undefined
+
+    if (licence.includes('Creative Commons Attribution')) return 1
+
+    for (const key of Object.keys(VIDEO_LICENCES)) {
+      const peertubeLicence = VIDEO_LICENCES[key]
+      if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10)
+    }
+
+    return undefined
+  }
+
+  private getCategory (categories: string[]) {
+    if (!categories) return undefined
+
+    const categoryString = categories[0]
+    if (!categoryString || typeof categoryString !== 'string') return undefined
+
+    if (categoryString === 'News & Politics') return 11
+
+    for (const key of Object.keys(VIDEO_CATEGORIES)) {
+      const category = VIDEO_CATEGORIES[key]
+      if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10)
+    }
+
+    return undefined
+  }
+
+  private getLanguage (language: string) {
+    return VIDEO_LANGUAGES[language] ? language : undefined
+  }
+}

server/core/helpers/youtube-dl/youtube-dl-wrapper.ts

@@ -0,0 +1,248 @@
+import { VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { ExecaError } from 'execa'
+import { move, pathExists, remove } from 'fs-extra/esm'
+import { readdir } from 'fs/promises'
+import { dirname, join } from 'path'
+import { inspect } from 'util'
+import { isVideoFileExtnameValid } from '../custom-validators/videos.js'
+import { t } from '../i18n.js'
+import { logger, loggerTagsFactory } from '../logger.js'
+import { generateVideoImportTmpPath } from '../utils.js'
+import { YoutubeDLCLI } from './youtube-dl-cli.js'
+import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder.js'
+
+const lTags = loggerTagsFactory('youtube-dl')
+
+export const YoutubeDlImportErrorCode = {
+  FETCH_ERROR: 0,
+  NOT_ONLY_UNICAST_URL: 1,
+  SKIP_PUBLICATION_DATE: 2,
+  IS_LIVE: 3
+}
+
+export type YoutubeDlImportErrorCodeType = typeof YoutubeDlImportErrorCode[keyof typeof YoutubeDlImportErrorCode]
+
+export class YoutubeDlImportError extends Error {
+  static fromError (options: {
+    err: Error
+    code: YoutubeDlImportErrorCodeType
+    message?: string
+  }) {
+    const { err, code, message } = options
+
+    const ytDlErr = new this({ message: message ?? err.message, code })
+    ytDlErr.cause = err
+
+    return ytDlErr
+  }
+
+  code: YoutubeDlImportErrorCodeType
+
+  constructor ({ message, code }) {
+    super(message)
+
+    this.code = code
+  }
+
+  isUnavailableVideoError () {
+    const stderr = this.getStderr()
+
+    if (stderr.includes('Video unavailable') || stderr.includes(' 429 ')) {
+      return true
+    }
+
+    return false
+  }
+
+  isRateLimitError () {
+    const stderr = this.getStderr()
+
+    if (stderr.includes('Sign in to confirm you’re not a bot')) {
+      return true
+    }
+
+    return false
+  }
+
+  private getStderr () {
+    const stderr = (this.cause as ExecaError)?.stderr
+
+    if (typeof stderr === 'string') return stderr
+
+    return ''
+  }
+}
+
+export type YoutubeDLSubs = {
+  language: string
+  filename: string
+  path: string
+}[]
+
+const processOptions = {
+  maxBuffer: 1024 * 1024 * 30 // 30MB
+}
+
+export class YoutubeDLWrapper {
+  constructor (
+    private readonly url: string,
+    private readonly enabledResolutions: VideoResolutionType[],
+    private readonly useBestFormat: boolean
+  ) {
+  }
+
+  async getInfoForDownload (options: {
+    userLanguage: string
+    youtubeDLArgs?: string[]
+  }): Promise<YoutubeDLInfo> {
+    const { userLanguage, youtubeDLArgs = [] } = options
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    try {
+      const info = await youtubeDL.getInfo({
+        url: this.url,
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
+        additionalYoutubeDLArgs: youtubeDLArgs,
+        processOptions
+      })
+
+      if (!info) {
+        throw new YoutubeDlImportError({
+          message: t(`Cannot fetch information from import for URL {targetUrl}`, userLanguage, { targetUrl: this.url }),
+          code: YoutubeDlImportErrorCode.FETCH_ERROR
+        })
+      }
+
+      if (info.is_live === true || [ 'is_live', 'post_live', 'is_upcoming' ].includes(info.live_status)) {
+        throw new YoutubeDlImportError({
+          message: t('Cannot download a live streaming for URL {targetUrl}', userLanguage, { targetUrl: this.url }),
+          code: YoutubeDlImportErrorCode.IS_LIVE
+        })
+      }
+
+      const infoBuilder = new YoutubeDLInfoBuilder(info)
+
+      return infoBuilder.getInfo()
+    } catch (err) {
+      throw YoutubeDlImportError.fromError({
+        err,
+        message: t(`Cannot get info from {targetUrl}`, userLanguage, { targetUrl: this.url }),
+        code: YoutubeDlImportErrorCode.FETCH_ERROR
+      })
+    }
+  }
+
+  async getInfoForListImport (options: {
+    userLanguage: string
+    latestVideosCount?: number
+  }) {
+    const { userLanguage } = options
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const list = await youtubeDL.getListInfo({
+      url: this.url,
+      latestVideosCount: options.latestVideosCount,
+      processOptions
+    })
+
+    if (!Array.isArray(list)) {
+      logger.debug(`List info from youtube-dl is not an array for ${this.url}.`, { info: inspect(list), ...lTags() })
+
+      throw new YoutubeDlImportError({
+        message: t(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`, userLanguage, { targetUrl: this.url }),
+        code: YoutubeDlImportErrorCode.FETCH_ERROR
+      })
+    }
+
+    return list.map(info => info.webpage_url)
+  }
+
+  async getSubtitles (): Promise<YoutubeDLSubs> {
+    const cwd = CONFIG.STORAGE.TMP_DIR
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } })
+    if (!files) return []
+
+    logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() })
+
+    const subtitles = files.reduce((acc, filename) => {
+      const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
+      if (!matched?.[1]) return acc
+
+      return [
+        ...acc,
+        {
+          language: matched[1],
+          path: join(cwd, filename),
+          filename
+        }
+      ]
+    }, [])
+
+    return subtitles
+  }
+
+  async downloadVideo (fileExt: string, timeout: number): Promise<string> {
+    // Leave empty the extension, youtube-dl will add it
+    const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
+
+    logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags())
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    try {
+      await youtubeDL.download({
+        url: this.url,
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
+        output: pathWithoutExtension,
+        timeout,
+        processOptions
+      })
+
+      // If youtube-dl did not guess an extension for our file, just use .mp4 as default
+      if (await pathExists(pathWithoutExtension)) {
+        await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
+      }
+
+      const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+      if (!path) {
+        const directoryContent = await readdir(dirname(pathWithoutExtension))
+
+        throw new Error(`Cannot guess path of ${pathWithoutExtension}. Directory content: ${directoryContent.join(', ')}`)
+      }
+
+      return path
+    } catch (err) {
+      this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+        .then(path => {
+          logger.debug('Error in youtube-dl import, deleting file if exists.', { err, pathToDelete: path, ...lTags() })
+
+          if (path) return remove(path)
+        })
+        .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
+
+      throw err
+    }
+  }
+
+  private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
+    if (!isVideoFileExtnameValid(sourceExt)) {
+      throw new Error('Invalid video extension ' + sourceExt)
+    }
+
+    const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
+
+    for (const extension of extensions) {
+      const path = tmpPath + extension
+
+      if (await pathExists(path)) return path
+    }
+
+    return undefined
+  }
+}