Init commit

server/scripts/create-generate-storyboard-job.ts

@@ -0,0 +1,80 @@
+import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
+import { StoryboardModel } from '@server/models/video/storyboard.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { program } from 'commander'
+
+program
+  .description('Generate videos storyboard')
+  .option('-v, --video [videoUUID]', 'Generate the storyboard of a specific video')
+  .option('-a, --all-videos', 'Generate missing storyboards of local videos')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (!options['video'] && !options['allVideos']) {
+  console.error('You need to choose videos for storyboard generation.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  JobQueue.Instance.init()
+
+  let ids: number[] = []
+
+  if (options['video']) {
+    const video = await VideoModel.load(toCompleteUUID(options['video']))
+
+    if (!video) {
+      console.error('Unknown video ' + options['video'])
+      process.exit(-1)
+    }
+
+    if (video.remote === true) {
+      console.error('Cannot process a remote video')
+      process.exit(-1)
+    }
+
+    if (video.isLive) {
+      console.error('Cannot process live video')
+      process.exit(-1)
+    }
+
+    ids.push(video.id)
+  } else {
+    ids = await listLocalMissingStoryboards()
+  }
+
+  for (const id of ids) {
+    const videoFull = await VideoModel.load(id)
+
+    if (videoFull.isLive) continue
+
+    await addLocalOrRemoteStoryboardJobIfNeeded({ video: videoFull, federate: true })
+
+    console.log(`Created generate-storyboard job for ${videoFull.name}.`)
+  }
+}
+
+async function listLocalMissingStoryboards () {
+  const ids = await VideoModel.listLocalIds()
+  const results: number[] = []
+
+  for (const id of ids) {
+    const storyboard = await StoryboardModel.loadByVideo(id)
+    if (!storyboard) results.push(id)
+  }
+
+  return results
+}

server/scripts/create-import-video-file-job.ts

@@ -0,0 +1,50 @@
+import { program } from 'commander'
+import { resolve } from 'path'
+import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
+import { initDatabaseModels } from '../core/initializers/database.js'
+import { JobQueue } from '../core/lib/job-queue/index.js'
+import { VideoModel } from '../core/models/video/video.js'
+
+program
+  .option('-v, --video [videoUUID]', 'Video UUID')
+  .option('-i, --import [videoFile]', 'Video file')
+  .description('Import a video file to replace an already uploaded file or to add a new resolution')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (options.video === undefined || options.import === undefined) {
+  console.error('All parameters are mandatory.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const uuid = toCompleteUUID(options.video)
+
+  if (isUUIDValid(uuid) === false) {
+    console.error('%s is not a valid video UUID.', options.video)
+    return
+  }
+
+  const video = await VideoModel.load(uuid)
+  if (!video) throw new Error('Video not found.')
+  if (video.isLocal() === false) throw new Error('Cannot import files of a non owned video.')
+
+  const dataInput = {
+    videoUUID: video.uuid,
+    filePath: resolve(options.import)
+  }
+
+  JobQueue.Instance.init()
+  await JobQueue.Instance.createJob({ type: 'video-file-import', payload: dataInput })
+  console.log('Import job for video %s created.', video.uuid)
+}

server/scripts/create-move-video-storage-job.ts

@@ -0,0 +1,133 @@
+import { createCommand } from '@commander-js/extra-typings'
+import { FileStorage, FileStorageType, VideoState } from '@peertube/peertube-models'
+import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import { hasVideoResourcesToBeMoved } from '@server/lib/move-storage/shared/move-video.js'
+import { moveToExternalStorageState, moveToFileSystemState } from '@server/lib/video-state.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideoFullLight } from '@server/types/models/index.js'
+
+const program = createCommand()
+  .description('Move videos to another storage.')
+  .option('-v, --video <videoUUID>', 'Move a specific video')
+  .option('-a, --all-videos', 'Migrate all videos')
+  .option('-o, --to-object-storage', 'Move videos in object storage')
+  .option('-f, --to-file-system', 'Move videos to file system')
+  .option('--force', 'Force the migration even if the video is already in a "Moving" state')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (!options.toObjectStorage && !options.toFileSystem) {
+  console.error('You need to choose where to send video files using --to-object-storage or --to-file-system.')
+  process.exit(-1)
+}
+
+if (!options.video && !options.allVideos) {
+  console.error('You need to choose which videos to move.')
+  process.exit(-1)
+}
+
+if (options.toObjectStorage && !CONFIG.OBJECT_STORAGE.ENABLED) {
+  console.error('Object storage is not enabled on this instance.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  JobQueue.Instance.init()
+
+  let ids: number[] = []
+
+  if (options.video) {
+    const video = await VideoModel.load(toCompleteUUID(options.video))
+
+    if (!video) {
+      console.error('Unknown video ' + options.video)
+      process.exit(-1)
+    }
+
+    if (video.remote === true) {
+      console.error('Cannot process a remote video')
+      process.exit(-1)
+    }
+
+    if (video.isLive) {
+      console.error('Cannot process live video')
+      process.exit(-1)
+    }
+
+    if (
+      options.force !== true &&
+      (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE || video.state === VideoState.TO_MOVE_TO_FILE_SYSTEM)
+    ) {
+      console.error('This video is already being moved to external storage/file system')
+      process.exit(-1)
+    }
+
+    ids.push(video.id)
+  } else {
+    ids = await VideoModel.listLocalIds()
+  }
+
+  for (const id of ids) {
+    const videoFull = await VideoModel.loadFull(id)
+    if (videoFull.isLive) continue
+
+    if (options.toObjectStorage) {
+      await createMoveJobIfNeeded({
+        video: videoFull,
+        targetStorage: FileStorage.OBJECT_STORAGE,
+        handler: () => moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined })
+      })
+
+      continue
+    }
+
+    if (options.toFileSystem) {
+      await createMoveJobIfNeeded({
+        video: videoFull,
+        targetStorage: FileStorage.FILE_SYSTEM,
+
+        handler: () => moveToFileSystemState({ video: videoFull, isNewVideo: false, transaction: undefined })
+      })
+    }
+  }
+}
+
+async function createMoveJobIfNeeded (options: {
+  video: MVideoFullLight
+  targetStorage: FileStorageType
+
+  handler: () => Promise<any>
+}) {
+  const { video, targetStorage, handler } = options
+
+  if (await hasVideoResourcesToBeMoved(video, targetStorage)) {
+    const type = targetStorage === FileStorage.OBJECT_STORAGE
+      ? 'to object storage'
+      : 'to file system'
+
+    console.log(`Moving ${type} video ${video.name}`)
+
+    const success = await handler()
+
+    if (!success) {
+      console.error(
+        `Cannot create move ${type} for ${video.name}: job creation may have failed or there may be pending transcoding jobs for this video`
+      )
+    } else {
+      console.log(`Created job ${type} for ${video.name}.`)
+    }
+  }
+}

server/scripts/house-keeping.ts

@@ -0,0 +1,92 @@
+import { createCommand } from '@commander-js/extra-typings'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { ActorImageModel } from '@server/models/actor/actor-image.js'
+import { ThumbnailModel } from '@server/models/video/thumbnail.js'
+import Bluebird from 'bluebird'
+import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js'
+
+const program = createCommand()
+  .description('Remove remote files')
+  .option('--delete-remote-files', 'Remove remote files (avatars, banners, thumbnails...)')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (!options.deleteRemoteFiles) {
+  console.log('At least one option must be set (for example --delete-remote-files).')
+  process.exit(0)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  displayPeerTubeMustBeStoppedWarning()
+
+  if (options.deleteRemoteFiles) {
+    return deleteRemoteFiles()
+  }
+}
+
+async function deleteRemoteFiles () {
+  console.log('Detecting remote files that can be deleted...')
+
+  const thumbnails = await ThumbnailModel.listRemoteOnDisk()
+  const actorImages = await ActorImageModel.listRemoteOnDisk()
+
+  if (thumbnails.length === 0 && actorImages.length === 0) {
+    console.log('No remote files to delete detected.')
+    process.exit(0)
+  }
+
+  const res = await askConfirmation(
+    `${thumbnails.length.toLocaleString()} thumbnails and ${actorImages.length.toLocaleString()} avatars/banners can be locally deleted. ` +
+    `PeerTube will download them again on-demand. ` +
+    `Do you want to delete these remote files?`
+  )
+
+  if (res !== true) {
+    console.log('Exiting without delete remote files.')
+    process.exit(0)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  console.log('Deleting remote thumbnails...')
+
+  await Bluebird.map(thumbnails, async thumbnail => {
+    if (!thumbnail.fileUrl) {
+      console.log(`Skipping thumbnail removal of ${thumbnail.getPath()} as we don't have its remote file URL in the database.`)
+      return
+    }
+
+    await thumbnail.removeThumbnail()
+
+    thumbnail.onDisk = false
+    await thumbnail.save()
+  }, { concurrency: 20 })
+
+  // ---------------------------------------------------------------------------
+
+  console.log('Deleting remote avatars/banners...')
+
+  await Bluebird.map(actorImages, async actorImage => {
+    if (!actorImage.fileUrl) {
+      console.log(`Skipping avatar/banner removal of ${actorImage.getPath()} as we don't have its remote file URL in the database.`)
+      return
+    }
+
+    await actorImage.removeImage()
+
+    actorImage.onDisk = false
+    await actorImage.save()
+  }, { concurrency: 20 })
+
+  console.log('Remote files deleted!')
+}

server/scripts/migrations/peertube-4.0.ts

@@ -0,0 +1,109 @@
+import Bluebird from 'bluebird'
+import { move } from 'fs-extra/esm'
+import { readFile, writeFile } from 'fs/promises'
+import { join } from 'path'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import {
+  generateHLSMasterPlaylistFilename,
+  generateHlsSha256SegmentsFilename,
+  getHLSResolutionPlaylistFilename
+} from '@server/lib/paths.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
+import { VideoModel } from '@server/models/video/video.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Migrate old HLS paths to new format.')
+
+  await initDatabaseModels(true)
+
+  JobQueue.Instance.init()
+
+  const ids = await VideoModel.listLocalIds()
+
+  await Bluebird.map(ids, async id => {
+    try {
+      await processVideo(id)
+    } catch (err) {
+      console.error('Cannot process video %s.', { err })
+    }
+  }, { concurrency: 5 })
+
+  console.log('Migration finished!')
+}
+
+async function processVideo (videoId: number) {
+  const video = await VideoModel.loadWithFiles(videoId)
+
+  const hls = video.getHLSPlaylist()
+  if (video.isLive || !hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
+    return
+  }
+
+  console.log(`Renaming HLS playlist files of video ${video.name}.`)
+
+  const playlist = await VideoStreamingPlaylistModel.loadHLSByVideo(video.id)
+  const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
+
+  const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
+  let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
+
+  for (const videoFile of hls.VideoFiles) {
+    const srcName = `${videoFile.resolution}.m3u8`
+    const dstName = getHLSResolutionPlaylistFilename(videoFile.filename)
+
+    const src = join(hlsDirPath, srcName)
+    const dst = join(hlsDirPath, dstName)
+
+    try {
+      await move(src, dst)
+
+      masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
+    } catch (err) {
+      console.error('Cannot move video file %s to %s.', src, dst, err)
+    }
+  }
+
+  await writeFile(masterPlaylistPath, masterPlaylistContent)
+
+  if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
+    try {
+      const newName = generateHlsSha256SegmentsFilename(video.isLive)
+
+      const dst = join(hlsDirPath, newName)
+      await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
+      playlist.segmentsSha256Filename = newName
+    } catch (err) {
+      console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
+    }
+  }
+
+  if (playlist.playlistFilename === 'master.m3u8') {
+    try {
+      const newName = generateHLSMasterPlaylistFilename(video.isLive)
+
+      const dst = join(hlsDirPath, newName)
+      await move(join(hlsDirPath, playlist.playlistFilename), dst)
+      playlist.playlistFilename = newName
+    } catch (err) {
+      console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
+    }
+  }
+
+  // Everything worked, we can save the playlist now
+  await playlist.save()
+
+  const allVideo = await VideoModel.loadFull(video.id)
+  await federateVideoIfNeeded(allVideo, false)
+
+  console.log(`Successfully moved HLS files of ${video.name}.`)
+}

server/scripts/migrations/peertube-4.2.ts

@@ -0,0 +1,122 @@
+import { maxBy, minBy } from '@peertube/peertube-core-utils'
+import { ActorImageType } from '@peertube/peertube-models'
+import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils'
+import { getImageSize, processImage } from '@server/helpers/image-utils.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { updateActorImages } from '@server/lib/activitypub/actors/index.js'
+import { sendUpdateActor } from '@server/lib/activitypub/send/index.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import { AccountModel } from '@server/models/account/account.js'
+import { ActorModel } from '@server/models/actor/actor.js'
+import { VideoChannelModel } from '@server/models/video/video-channel.js'
+import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models/index.js'
+import { join } from 'path'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Generate avatar miniatures from existing avatars.')
+
+  await initDatabaseModels(true)
+  JobQueue.Instance.init()
+
+  const accounts: AccountModel[] = await AccountModel.findAll({
+    include: [
+      {
+        model: ActorModel,
+        required: true,
+        where: {
+          serverId: null
+        }
+      },
+      {
+        model: VideoChannelModel,
+        include: [
+          {
+            model: AccountModel
+          }
+        ]
+      }
+    ]
+  })
+
+  for (const account of accounts) {
+    try {
+      await fillAvatarSizeIfNeeded(account)
+      await generateSmallerAvatarIfNeeded(account)
+    } catch (err) {
+      console.error(`Cannot process account avatar ${account.name}`, err)
+    }
+
+    for (const videoChannel of account.VideoChannels) {
+      try {
+        await fillAvatarSizeIfNeeded(videoChannel)
+        await generateSmallerAvatarIfNeeded(videoChannel)
+      } catch (err) {
+        console.error(`Cannot process channel avatar ${videoChannel.name}`, err)
+      }
+    }
+  }
+
+  console.log('Generation finished!')
+}
+
+async function fillAvatarSizeIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) {
+  const avatars = accountOrChannel.Actor.Avatars
+
+  for (const avatar of avatars) {
+    if (avatar.width && avatar.height) continue
+
+    console.log('Filling size of avatars of %s.', accountOrChannel.name)
+
+    const { width, height } = await getImageSize(join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, avatar.filename))
+    avatar.width = width
+    avatar.height = height
+
+    await avatar.save()
+  }
+}
+
+async function generateSmallerAvatarIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) {
+  const avatars = accountOrChannel.Actor.Avatars
+  if (avatars.length !== 1) {
+    return
+  }
+
+  console.log(`Processing ${accountOrChannel.name}.`)
+
+  await generateSmallerAvatar(accountOrChannel.Actor)
+  accountOrChannel.Actor = Object.assign(accountOrChannel.Actor, { Server: null })
+
+  return sendUpdateActor(accountOrChannel, undefined)
+}
+
+async function generateSmallerAvatar (actor: MActorDefault) {
+  const bigAvatar = maxBy(actor.Avatars, 'width')
+
+  const imageSize = minBy(ACTOR_IMAGES_SIZE[ActorImageType.AVATAR], 'width')
+  const sourceFilename = bigAvatar.filename
+
+  const newImageName = buildUUID() + getLowercaseExtension(sourceFilename)
+  const source = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, sourceFilename)
+  const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, newImageName)
+
+  await processImage({ path: source, destination, newSize: imageSize, keepOriginal: true })
+
+  const actorImageInfo = {
+    name: newImageName,
+    fileUrl: null,
+    height: imageSize.height,
+    width: imageSize.width,
+    onDisk: true
+  }
+
+  await updateActorImages(actor, ActorImageType.AVATAR, [ actorImageInfo ], undefined)
+}

server/scripts/migrations/peertube-5.0.ts

@@ -0,0 +1,71 @@
+import { ensureDir } from 'fs-extra/esm'
+import { Op } from 'sequelize'
+import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { DIRECTORIES } from '@server/initializers/constants.js'
+import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideoFullLight } from '@server/types/models/index.js'
+import { VideoPrivacy } from '@peertube/peertube-models'
+import { initDatabaseModels } from '@server/initializers/database.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Moving private video files in dedicated folders.')
+
+  await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)
+  await ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE)
+
+  await initDatabaseModels(true)
+
+  const videos = await VideoModel.unscoped().findAll({
+    attributes: [ 'uuid' ],
+    where: {
+      privacy: {
+        [Op.in]: [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]
+      }
+    }
+  })
+
+  for (const { uuid } of videos) {
+    try {
+      console.log('Moving files of video %s.', uuid)
+
+      const video = await VideoModel.loadFull(uuid)
+
+      try {
+        await moveFilesIfPrivacyChanged(video, VideoPrivacy.PUBLIC)
+      } catch (err) {
+        console.error('Cannot move files of video %s.', uuid, err)
+      }
+
+      try {
+        await updateTorrents(video)
+      } catch (err) {
+        console.error('Cannot regenerate torrents of video %s.', uuid, err)
+      }
+    } catch (err) {
+      console.error('Cannot process video %s.', uuid, err)
+    }
+  }
+}
+
+async function updateTorrents (video: MVideoFullLight) {
+  for (const file of video.VideoFiles) {
+    await updateTorrentMetadata(video, file)
+
+    await file.save()
+  }
+
+  const playlist = video.getHLSPlaylist()
+  for (const file of (playlist?.VideoFiles || [])) {
+    await updateTorrentMetadata(playlist, file)
+
+    await file.save()
+  }
+}

server/scripts/migrations/peertube-6.3.ts

@@ -0,0 +1,133 @@
+import { ffprobePromise, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
+import { VideoFileStream } from '@peertube/peertube-models'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { buildFileMetadata } from '@server/lib/video-file.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoModel } from '@server/models/video/video.js'
+import Bluebird from 'bluebird'
+import { pathExists } from 'fs-extra/esm'
+import { QueryTypes } from 'sequelize'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  {
+    console.log('## Updating "formatFlags" column for web videos in "videoFile" table in database ##\n')
+
+    const totalQuery = 'SELECT COUNT(*) AS "total" FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
+    const res = await sequelizeTypescript.query<{ total: string }>(totalQuery, { type: QueryTypes.SELECT as QueryTypes.SELECT })
+    const total = parseInt(res[0].total)
+
+    console.log(`Will update ${total.toLocaleString()} rows`)
+
+    if (total > 10000) {
+      console.log('Processing update in chunks because there are many rows to update...')
+
+      const chunkSize = 10000
+      let remaining = total
+
+      while (remaining > chunkSize) {
+        const before = new Date().getTime()
+
+        await sequelizeTypescript.query(
+          'UPDATE "videoFile" SET "formatFlags" = 1 WHERE id IN (' +
+            'SELECT id FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1 LIMIT ' + chunkSize +
+          ')'
+        )
+
+        remaining -= chunkSize
+
+        const ms = new Date().getTime() - before
+        console.log(`Processed ${chunkSize.toLocaleString()} rows in ${ms.toLocaleString()}ms. Remaining: ${remaining.toLocaleString()}`)
+      }
+    }
+
+    const query = 'UPDATE "videoFile" SET "formatFlags" = 1 WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
+    await sequelizeTypescript.query(query)
+
+    console.log('Rows updated!')
+  }
+
+  console.log('## Assigning metadata information to local video files ##\n')
+
+  const ids = await VideoModel.listLocalIds()
+
+  await Bluebird.map(ids, async id => {
+    try {
+      await processVideo(id)
+    } catch (err) {
+      console.error('Cannot process video ' + id, err)
+    }
+  }, { concurrency: 5 })
+
+  console.log('\n## Migration finished! ##')
+}
+
+async function processVideo (videoId: number) {
+  const video = await VideoModel.loadWithFiles(videoId)
+  if (video.isLive) return
+
+  const files = await Promise.all(video.getAllFiles().map(f => VideoFileModel.loadWithMetadata(f.id)))
+
+  if (!files.some(f => f.fps === -1 || !f.metadata || !f.streams || f.width === null || f.height === null)) return
+
+  console.log('Processing video "' + video.name + '"')
+
+  for (const file of files) {
+    if (!file.metadata || file.fps === -1) {
+      const fileWithVideoOrPlaylist = file.videoStreamingPlaylistId
+        ? file.withVideoOrPlaylist(video.getHLSPlaylist())
+        : file.withVideoOrPlaylist(video)
+
+      await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async path => {
+        if (!await pathExists(path)) {
+          console.error(
+            `Skipping processing file ${file.id} because ${path} does not exist on disk for video "${video.name}" (uuid: ${video.uuid})`
+          )
+          return
+        }
+
+        console.log(`Filling metadata field of video "${video.name}" - file ${file.id} because it is missing in database`)
+
+        const probe = await ffprobePromise(path)
+
+        file.metadata = await buildFileMetadata(path, probe)
+        file.fps = await getVideoStreamFPS(path, probe)
+
+        await file.save()
+      })
+    }
+
+    if (!file.metadata) continue
+
+    file.streams = VideoFileStream.NONE
+
+    const videoStream = file.metadata.streams.find(s => s.codec_type === 'video')
+
+    if (videoStream) {
+      file.streams |= VideoFileStream.VIDEO
+
+      file.width = videoStream.width
+      file.height = videoStream.height
+    } else {
+      file.width = 0
+      file.height = 0
+    }
+
+    if (file.metadata.streams.some(s => s.codec_type === 'audio')) {
+      file.streams |= VideoFileStream.AUDIO
+    }
+
+    await file.save()
+  }
+
+  console.log('Successfully processed video "' + video.name + '"')
+}

server/scripts/migrations/peertube-7.2.ts

@@ -0,0 +1,20 @@
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  console.log('Running SQL request to update comments count...')
+  {
+    const query = 'UPDATE "video" SET "comments" = (SELECT COUNT(*) FROM "videoComment" WHERE "videoComment"."videoId" = "video"."id")'
+    await sequelizeTypescript.query(query)
+  }
+
+  console.log('Done!')
+}

server/scripts/migrations/peertube-8.0.ts

@@ -0,0 +1,45 @@
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { ApplicationModel } from '@server/models/application/application.js'
+import { QueryTypes } from 'sequelize'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const application = await ApplicationModel.load()
+  if (application.migrationVersion < 960) {
+    console.error('This migration script must be run after v8.0 migration')
+    process.exit(-1)
+  }
+
+  const chunkSize = 500
+  let i = 1
+
+  console.log(`Running SQL request to delete unused actors, in chunks of maximum ${chunkSize} rows...`)
+
+  while (true) {
+    console.log('Processing chunk ' + i)
+
+    const query = `WITH cte AS (SELECT id FROM "actor" WHERE "accountId" IS NULL AND "videoChannelId" IS NULL LIMIT ${chunkSize}) ` +
+      `DELETE FROM "actor" WHERE id IN (SELECT "id" FROM cte) RETURNING id`
+
+    const deleted = await sequelizeTypescript.query(query, {
+      type: QueryTypes.BULKDELETE,
+      replacements: { limit: chunkSize }
+    })
+
+    if (deleted < chunkSize) {
+      break
+    }
+
+    i++
+  }
+
+  console.log('Done!')
+}

server/scripts/parse-log.ts

@@ -0,0 +1,161 @@
+import { program } from 'commander'
+import { createReadStream } from 'fs'
+import { readdir } from 'fs/promises'
+import { join } from 'path'
+import { stdin } from 'process'
+import { createInterface } from 'readline'
+import { format as sqlFormat } from 'sql-formatter'
+import { inspect } from 'util'
+import * as winston from 'winston'
+import { labelFormatter, mtimeSortFilesDesc } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+
+program
+  .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
+  .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)')
+  .option('-t, --tags [tags...]', 'Display only lines with these tags')
+  .option('-n, --not-tags [tags...]', 'Do not display lines containing these tags')
+  .parse(process.argv)
+
+const options = program.opts()
+
+const excludedKeys = {
+  level: true,
+  message: true,
+  splat: true,
+  timestamp: true,
+  tags: true,
+  label: true,
+  sql: true
+}
+function keysExcluder (key, value) {
+  return excludedKeys[key] === true ? undefined : value
+}
+
+const loggerFormat = winston.format.printf((info) => {
+  let additionalInfos = JSON.stringify(info, keysExcluder, 2)
+  if (additionalInfos === '{}') additionalInfos = ''
+  else additionalInfos = ' ' + additionalInfos
+
+  if (info.sql) {
+    if (CONFIG.LOG.PRETTIFY_SQL) {
+      additionalInfos += '\n' + sqlFormat(info.sql as string, {
+        language: 'sql',
+        tabWidth: 2
+      })
+    } else {
+      additionalInfos += ' - ' + info.sql
+    }
+  }
+
+  return `[${info.label}] ${toTimeFormat(info.timestamp as string)} ${info.level}: ${info.message}${additionalInfos}`
+})
+
+const logger = winston.createLogger({
+  transports: [
+    new winston.transports.Console({
+      level: options.level || 'debug',
+      stderrLevels: [],
+      format: winston.format.combine(
+        winston.format.splat(),
+        labelFormatter(),
+        winston.format.colorize(),
+        loggerFormat
+      )
+    })
+  ],
+  exitOnError: true
+})
+
+const logLevels = {
+  error: logger.error.bind(logger),
+  warn: logger.warn.bind(logger),
+  info: logger.info.bind(logger),
+  debug: logger.debug.bind(logger)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => console.error(err))
+
+async function run () {
+  const files = await getFiles()
+
+  for (const file of files) {
+    if (file === 'peertube-audit.log') continue
+
+    await readFile(file)
+  }
+}
+
+function readFile (file: string) {
+  console.log('Opening %s.', file)
+
+  const stream = file === '-' ? stdin : createReadStream(file)
+
+  const rl = createInterface({
+    input: stream
+  })
+
+  return new Promise<void>(res => {
+    rl.on('line', line => {
+      try {
+        const log = JSON.parse(line)
+        if (options.tags && !containsTags(log.tags, options.tags)) {
+          return
+        }
+
+        if (options.notTags && containsTags(log.tags, options.notTags)) {
+          return
+        }
+
+        // Don't know why but loggerFormat does not remove splat key
+        Object.assign(log, { splat: undefined })
+
+        logLevels[log.level](log)
+      } catch (err) {
+        console.error('Cannot parse line.', inspect(line))
+        throw err
+      }
+    })
+
+    stream.once('end', () => res())
+  })
+}
+
+// Thanks: https://stackoverflow.com/a/37014317
+async function getNewestFile (files: string[], basePath: string) {
+  const sorted = await mtimeSortFilesDesc(files, basePath)
+
+  return (sorted.length > 0) ? sorted[0].file : ''
+}
+
+async function getFiles () {
+  if (options.files) return options.files
+
+  const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR)
+
+  const filename = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR)
+  return [ join(CONFIG.STORAGE.LOG_DIR, filename) ]
+}
+
+function toTimeFormat (time: string) {
+  const timestamp = Date.parse(time)
+
+  if (isNaN(timestamp) === true) return 'Unknown date'
+
+  const d = new Date(timestamp)
+  return d.toLocaleString() + `.${d.getMilliseconds()}`
+}
+
+function containsTags (loggerTags: string[], optionsTags: string[]) {
+  if (!loggerTags) return false
+
+  for (const lt of loggerTags) {
+    for (const ot of optionsTags) {
+      if (lt === ot) return true
+    }
+  }
+
+  return false
+}

server/scripts/plugin/install.ts

@@ -0,0 +1,41 @@
+import { program } from 'commander'
+import { isAbsolute } from 'path'
+import { initDatabaseModels } from '../../core/initializers/database.js'
+import { PluginManager } from '../../core/lib/plugins/plugin-manager.js'
+
+program
+  .option('-n, --npm-name [npmName]', 'Plugin to install')
+  .option('-v, --plugin-version [pluginVersion]', 'Plugin version to install')
+  .option('-p, --plugin-path [pluginPath]', 'Path of the plugin you want to install')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (!options.npmName && !options.pluginPath) {
+  console.error('You need to specify a plugin name with the desired version, or a plugin path.')
+  process.exit(-1)
+}
+
+if (options.pluginPath && !isAbsolute(options.pluginPath)) {
+  console.error('Plugin path should be absolute.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const toInstall = options.npmName || options.pluginPath
+  await PluginManager.Instance.install({
+    toInstall,
+    version: options.pluginVersion,
+    fromDisk: !!options.pluginPath,
+    register: false
+  })
+}

server/scripts/plugin/uninstall.ts

@@ -0,0 +1,29 @@
+import { program } from 'commander'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { PluginManager } from '@server/lib/plugins/plugin-manager.js'
+
+program
+  .option('-n, --npm-name [npmName]', 'Package name to install')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (!options.npmName) {
+  console.error('You need to specify the plugin name.')
+  process.exit(-1)
+}
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+
+  await initDatabaseModels(true)
+
+  const toUninstall = options.npmName
+  await PluginManager.Instance.uninstall({ npmName: toUninstall, unregister: false })
+}

server/scripts/prune-storage.ts

@@ -0,0 +1,348 @@
+import { createCommand } from '@commander-js/extra-typings'
+import { uniqify } from '@peertube/peertube-core-utils'
+import { FileStorage, ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models'
+import { DIRECTORIES, USER_EXPORT_FILE_PREFIX } from '@server/initializers/constants.js'
+import { listKeysOfPrefix, removeObjectByFullKey } from '@server/lib/object-storage/object-storage-helpers.js'
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { StoryboardModel } from '@server/models/video/storyboard.js'
+import { VideoCaptionModel } from '@server/models/video/video-caption.js'
+import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoSourceModel } from '@server/models/video/video-source.js'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
+import Bluebird from 'bluebird'
+import { remove } from 'fs-extra/esm'
+import { readdir, stat } from 'fs/promises'
+import { basename, dirname, join } from 'path'
+import { getUUIDFromFilename } from '../core/helpers/utils.js'
+import { CONFIG } from '../core/initializers/config.js'
+import { initDatabaseModels } from '../core/initializers/database.js'
+import { ActorImageModel } from '../core/models/actor/actor-image.js'
+import { VideoRedundancyModel } from '../core/models/redundancy/video-redundancy.js'
+import { ThumbnailModel } from '../core/models/video/thumbnail.js'
+import { VideoModel } from '../core/models/video/video.js'
+import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js'
+
+const program = createCommand()
+  .description('Remove unused local objects (video files, captions, user exports...) from object storage or file system')
+  .option('-y, --yes', 'Auto confirm files deletion')
+  .parse(process.argv)
+
+const options = program.opts()
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  displayPeerTubeMustBeStoppedWarning()
+
+  await new FSPruner().prune()
+
+  console.log('\n')
+
+  await new ObjectStoragePruner().prune()
+}
+
+// ---------------------------------------------------------------------------
+// Object storage
+// ---------------------------------------------------------------------------
+
+class ObjectStoragePruner {
+  private readonly keysToDelete: { bucket: string, key: string }[] = []
+
+  async prune () {
+    if (!CONFIG.OBJECT_STORAGE.ENABLED) return
+
+    console.log('Pruning object storage.')
+
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.WEB_VIDEOS, this.doesWebVideoFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, this.doesStreamingPlaylistFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES, this.doesOriginalFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.USER_EXPORTS, this.doesUserExportFileExistFactory())
+    await this.findFilesToDelete(CONFIG.OBJECT_STORAGE.CAPTIONS, this.doesCaptionFileExistFactory())
+
+    if (this.keysToDelete.length === 0) {
+      console.log('No unknown object storage files to delete.')
+      return
+    }
+
+    const formattedKeysToDelete = this.keysToDelete.map(({ bucket, key }) => ` In bucket ${bucket}: ${key}`).join('\n')
+    console.log(`${this.keysToDelete.length} unknown files from object storage can be deleted:\n${formattedKeysToDelete}\n`)
+
+    const res = await askPruneConfirmation(options.yes)
+    if (res !== true) {
+      console.log('Exiting without deleting object storage files.')
+      return
+    }
+
+    console.log('Deleting object storage files...\n')
+
+    for (const { bucket, key } of this.keysToDelete) {
+      await removeObjectByFullKey(key, { BUCKET_NAME: bucket })
+    }
+
+    console.log(`${this.keysToDelete.length} object storage files deleted.`)
+  }
+
+  private async findFilesToDelete (
+    config: { BUCKET_NAME: string, PREFIX?: string },
+    existFun: (file: string) => Promise<boolean> | boolean
+  ) {
+    try {
+      const keys = await listKeysOfPrefix('', config)
+
+      await Bluebird.map(keys, async key => {
+        if (await existFun(key) !== true) {
+          this.keysToDelete.push({ bucket: config.BUCKET_NAME, key })
+        }
+      }, { concurrency: 20 })
+    } catch (err) {
+      const prefixMessage = config.PREFIX
+        ? ` and prefix ${config.PREFIX}`
+        : ''
+
+      console.error('Cannot find files to delete in bucket ' + config.BUCKET_NAME + prefixMessage, { err })
+    }
+  }
+
+  private doesWebVideoFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
+
+      return VideoFileModel.doesOwnedWebVideoFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesStreamingPlaylistFileExistFactory () {
+    return (key: string) => {
+      const sanitizedKey = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+      const uuid = dirname(sanitizedKey).replace(/^hls\//, '')
+
+      return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(uuid, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesOriginalFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
+
+      return VideoSourceModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesUserExportFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
+
+      return UserExportModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private doesCaptionFileExistFactory () {
+    return (key: string) => {
+      const filename = this.sanitizeKey(key, CONFIG.OBJECT_STORAGE.CAPTIONS)
+
+      return VideoCaptionModel.doesOwnedFileExist(filename, FileStorage.OBJECT_STORAGE)
+    }
+  }
+
+  private sanitizeKey (key: string, config: { PREFIX: string }) {
+    return key.replace(new RegExp(`^${config.PREFIX}`), '')
+  }
+}
+
+// ---------------------------------------------------------------------------
+// FS
+// ---------------------------------------------------------------------------
+
+class FSPruner {
+  private pathsToDelete: string[] = []
+
+  async prune () {
+    const dirs = Object.values(CONFIG.STORAGE)
+
+    if (uniqify(dirs).length !== dirs.length) {
+      console.error('Cannot prune storage because you put multiple storage keys in the same directory.')
+      process.exit(0)
+    }
+
+    console.log('Pruning filesystem storage.')
+
+    console.log('Detecting files to remove, it can take a while...')
+
+    await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PUBLIC, this.doesWebVideoFileExistFactory())
+    await this.findFilesToDelete(DIRECTORIES.WEB_VIDEOS.PRIVATE, this.doesWebVideoFileExistFactory())
+
+    await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, this.doesHLSPlaylistExistFactory())
+    await this.findFilesToDelete(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, this.doesHLSPlaylistExistFactory())
+
+    await this.findFilesToDelete(DIRECTORIES.ORIGINAL_VIDEOS, this.doesOriginalVideoExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.TORRENTS_DIR, this.doesTorrentFileExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.REDUNDANCY_DIR, this.doesRedundancyExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.PREVIEWS_DIR, this.doesThumbnailExistFactory(true, ThumbnailType.PREVIEW))
+    await this.findFilesToDelete(CONFIG.STORAGE.THUMBNAILS_DIR, this.doesThumbnailExistFactory(false, ThumbnailType.MINIATURE))
+
+    await this.findFilesToDelete(CONFIG.STORAGE.CAPTIONS_DIR, this.doesCaptionExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.STORYBOARDS_DIR, this.doesStoryboardExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.doesActorImageExistFactory())
+
+    await this.findFilesToDelete(CONFIG.STORAGE.TMP_PERSISTENT_DIR, this.doesUserExportExistFactory())
+
+    const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR)
+    this.pathsToDelete = [ ...this.pathsToDelete, ...tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t)) ]
+
+    if (this.pathsToDelete.length === 0) {
+      console.log('No unknown filesystem files to delete.')
+      return
+    }
+
+    const formattedKeysToDelete = this.pathsToDelete.map(p => ` ${p}`).join('\n')
+    console.log(`${this.pathsToDelete.length} unknown files from filesystem can be deleted:\n${formattedKeysToDelete}\n`)
+
+    const res = await askPruneConfirmation(options.yes)
+    if (res !== true) {
+      console.log('Exiting without deleting filesystem files.')
+      return
+    }
+
+    console.log('Deleting filesystem files...\n')
+
+    for (const path of this.pathsToDelete) {
+      await remove(path)
+    }
+
+    console.log(`${this.pathsToDelete.length} filesystem files deleted.`)
+  }
+
+  private async findFilesToDelete (directory: string, existFun: (file: string) => Promise<boolean> | boolean) {
+    const files = await readdir(directory)
+
+    await Bluebird.map(files, async file => {
+      const filePath = join(directory, file)
+
+      if (await existFun(filePath) !== true) {
+        this.pathsToDelete.push(filePath)
+      }
+    }, { concurrency: 20 })
+  }
+
+  private doesWebVideoFileExistFactory () {
+    return (filePath: string) => {
+      // Don't delete private directory
+      if (filePath === DIRECTORIES.WEB_VIDEOS.PRIVATE) return true
+
+      return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
+    }
+  }
+
+  private doesHLSPlaylistExistFactory () {
+    return (hlsPath: string) => {
+      // Don't delete private directory
+      if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true
+
+      return VideoStreamingPlaylistModel.doesOwnedVideoUUIDExist(basename(hlsPath), FileStorage.FILE_SYSTEM)
+    }
+  }
+
+  private doesOriginalVideoExistFactory () {
+    return (filePath: string) => {
+      return VideoSourceModel.doesOwnedFileExist(basename(filePath), FileStorage.FILE_SYSTEM)
+    }
+  }
+
+  private doesTorrentFileExistFactory () {
+    return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath))
+  }
+
+  private doesThumbnailExistFactory (keepOnlyOwned: boolean, type: ThumbnailType_Type) {
+    return async (filePath: string) => {
+      const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type)
+      if (!thumbnail) return false
+
+      if (keepOnlyOwned) {
+        const video = await VideoModel.load(thumbnail.videoId)
+        if (video.isLocal() === false) return false
+      }
+
+      return true
+    }
+  }
+
+  private doesActorImageExistFactory () {
+    return async (filePath: string) => {
+      const image = await ActorImageModel.loadByFilename(basename(filePath))
+
+      return !!image
+    }
+  }
+
+  private doesStoryboardExistFactory () {
+    return async (filePath: string) => {
+      const storyboard = await StoryboardModel.loadByFilename(basename(filePath))
+
+      return !!storyboard
+    }
+  }
+
+  private doesCaptionExistFactory () {
+    return async (filePath: string) => {
+      const caption = await VideoCaptionModel.loadWithVideoByFilename(basename(filePath))
+
+      return !!caption
+    }
+  }
+
+  private doesRedundancyExistFactory () {
+    return async (filePath: string) => {
+      const isPlaylist = (await stat(filePath)).isDirectory()
+
+      if (isPlaylist) {
+        // Don't delete HLS redundancy directory
+        if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true
+
+        const uuid = getUUIDFromFilename(filePath)
+        const video = await VideoModel.loadWithFiles(uuid)
+        if (!video) return false
+
+        const p = video.getHLSPlaylist()
+        if (!p) return false
+
+        const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id)
+        return !!redundancy
+      }
+
+      // WebTorrent support redundancy has been removed from PeerTube
+      return false
+    }
+  }
+
+  private doesUserExportExistFactory () {
+    return (filePath: string) => {
+      const filename = basename(filePath)
+
+      // Only detect non-existing user export
+      if (!filename.startsWith(USER_EXPORT_FILE_PREFIX)) return true
+
+      return UserExportModel.doesOwnedFileExist(filename, FileStorage.FILE_SYSTEM)
+    }
+  }
+}
+
+async function askPruneConfirmation (yes?: boolean) {
+  if (yes === true) return true
+
+  return askConfirmation(
+    'These unknown files can be deleted, but please check your backups first (bugs happen). ' +
+      'Can we delete these files?'
+  )
+}

server/scripts/regenerate-thumbnails.ts

@@ -0,0 +1,64 @@
+import Bluebird from 'bluebird'
+import { program } from 'commander'
+import { pathExists, remove } from 'fs-extra/esm'
+import { generateImageFilename, processImage } from '@server/helpers/image-utils.js'
+import { THUMBNAILS_SIZE } from '@server/initializers/constants.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { VideoModel } from '@server/models/video/video.js'
+
+program
+  .description('Regenerate local thumbnails using preview files')
+  .parse(process.argv)
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => console.error(err))
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const ids = await VideoModel.listLocalIds()
+
+  await Bluebird.map(ids, id => {
+    return processVideo(id)
+      .catch(err => console.error('Cannot process video %d.', id, err))
+  }, { concurrency: 20 })
+}
+
+async function processVideo (id: number) {
+  const video = await VideoModel.loadWithFiles(id)
+
+  console.log('Processing video %s.', video.name)
+
+  const thumbnail = video.getMiniature()
+  const preview = video.getPreview()
+
+  const previewPath = preview.getPath()
+
+  if (!await pathExists(previewPath)) {
+    throw new Error(`Preview ${previewPath} does not exist on disk`)
+  }
+
+  const size = {
+    width: THUMBNAILS_SIZE.width,
+    height: THUMBNAILS_SIZE.height
+  }
+
+  const oldPath = thumbnail.getPath()
+
+  // Update thumbnail
+  thumbnail.filename = generateImageFilename()
+  thumbnail.width = size.width
+  thumbnail.height = size.height
+
+  const thumbnailPath = thumbnail.getPath()
+  await processImage({ path: previewPath, destination: thumbnailPath, newSize: size, keepOriginal: true })
+
+  // Save new attributes
+  await thumbnail.save()
+
+  // Remove old thumbnail
+  await remove(oldPath)
+
+  // Don't federate, remote instances will refresh the thumbnails after a while
+}

server/scripts/reset-password.ts

@@ -0,0 +1,58 @@
+import { program } from 'commander'
+import readline from 'readline'
+import { Writable } from 'stream'
+import { isUserPasswordValid } from '@server/helpers/custom-validators/users.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { UserModel } from '@server/models/user/user.js'
+
+program
+  .option('-u, --user [user]', 'User')
+  .parse(process.argv)
+
+const options = program.opts()
+
+if (options.user === undefined) {
+  console.error('All parameters are mandatory.')
+  process.exit(-1)
+}
+
+initDatabaseModels(true)
+  .then(() => {
+    return UserModel.loadByUsername(options.user)
+  })
+  .then(user => {
+    if (!user) {
+      console.error('Unknown user.')
+      process.exit(-1)
+    }
+
+    const mutableStdout = new Writable({
+      write: function (_chunk, _encoding, callback) {
+        callback()
+      }
+    })
+    const rl = readline.createInterface({
+      input: process.stdin,
+      output: mutableStdout,
+      terminal: true
+    })
+
+    console.log('New password?')
+    rl.on('line', function (password) {
+      if (!isUserPasswordValid(password)) {
+        console.error('New password is invalid.')
+        process.exit(-1)
+      }
+
+      user.password = password
+
+      user.save()
+        .then(() => console.log('User password updated.'))
+        .catch(err => console.error(err))
+        .finally(() => process.exit(0))
+    })
+  })
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })

server/scripts/shared/common.ts

@@ -0,0 +1,30 @@
+import prompt from 'prompt'
+
+export async function askConfirmation (message: string) {
+  return new Promise((res, rej) => {
+    prompt.start()
+
+    const schema = {
+      properties: {
+        confirm: {
+          type: 'string',
+          description: message + ' (y/n)',
+          default: 'n',
+          validator: /y[es]*|n[o]?/,
+          warning: 'Must respond yes or no',
+          required: true
+        }
+      }
+    }
+
+    prompt.get(schema, function (err, result) {
+      if (err) return rej(err)
+
+      return res(result.confirm?.match(/y/) !== null)
+    })
+  })
+}
+
+export function displayPeerTubeMustBeStoppedWarning () {
+  console.log(`/!\\ PeerTube must be stopped before running this script /!\\\n`)
+}

server/scripts/update-host.ts

@@ -0,0 +1,194 @@
+import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { getServerActor } from '@server/models/application/application.js'
+import { WEBSERVER } from '@server/initializers/constants.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import {
+  getLocalAccountActivityPubUrl,
+  getLocalVideoActivityPubUrl,
+  getLocalVideoAnnounceActivityPubUrl,
+  getLocalVideoChannelActivityPubUrl,
+  getLocalVideoCommentActivityPubUrl,
+  getLocalVideoPlaylistActivityPubUrl,
+  getLocalVideoPlaylistElementActivityPubUrl
+} from '@server/lib/activitypub/url.js'
+import { AccountModel } from '@server/models/account/account.js'
+import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
+import { ActorModel } from '@server/models/actor/actor.js'
+import { VideoChannelModel } from '@server/models/video/video-channel.js'
+import { VideoCommentModel } from '@server/models/video/video-comment.js'
+import { VideoShareModel } from '@server/models/video/video-share.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MActorAccount } from '@server/types/models/index.js'
+import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
+import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const serverAccount = await getServerActor()
+
+  {
+    const res = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined, 0, 1)
+    const hasFollowing = res.total > 0
+
+    if (hasFollowing === true) {
+      throw new Error('Cannot update host because you follow other servers!')
+    }
+  }
+
+  console.log('Updating actors.')
+
+  const actors: MActorAccount[] = await ActorModel.unscoped().findAll({
+    where: {
+      serverId: null
+    },
+    include: [
+      {
+        model: VideoChannelModel.unscoped(),
+        required: false
+      },
+      {
+        model: AccountModel.unscoped(),
+        required: false
+      }
+    ]
+  })
+  for (const actor of actors) {
+    console.log('Updating actor ' + actor.url)
+
+    const newUrl = actor.Account
+      ? getLocalAccountActivityPubUrl(actor.preferredUsername)
+      : getLocalVideoChannelActivityPubUrl(actor.preferredUsername)
+
+    actor.url = newUrl
+    actor.inboxUrl = newUrl + '/inbox'
+    actor.outboxUrl = newUrl + '/outbox'
+    actor.sharedInboxUrl = WEBSERVER.URL + '/inbox'
+    actor.followersUrl = newUrl + '/followers'
+    actor.followingUrl = newUrl + '/following'
+
+    await actor.save()
+  }
+
+  console.log('Updating video shares.')
+
+  const videoShares: VideoShareModel[] = await VideoShareModel.findAll({
+    include: [
+      {
+        model: VideoModel.unscoped(),
+        where: {
+          remote: false
+        },
+        required: true
+      },
+      ActorModel.unscoped()
+    ]
+  })
+  for (const videoShare of videoShares) {
+    console.log('Updating video share ' + videoShare.url)
+
+    videoShare.url = getLocalVideoAnnounceActivityPubUrl(videoShare.Actor, videoShare.Video)
+    await videoShare.save()
+  }
+
+  console.log('Updating video comments.')
+  const videoComments: VideoCommentModel[] = await VideoCommentModel.findAll({
+    include: [
+      {
+        model: VideoModel.unscoped()
+      },
+      {
+        model: AccountModel.unscoped(),
+        required: true,
+        include: [
+          {
+            model: ActorModel.unscoped(),
+            where: {
+              serverId: null
+            },
+            required: true
+          }
+        ]
+      }
+    ]
+  })
+  for (const comment of videoComments) {
+    console.log('Updating comment ' + comment.url)
+
+    comment.url = getLocalVideoCommentActivityPubUrl(comment.Video, comment)
+    await comment.save()
+  }
+
+  console.log('Updating video playlists.')
+  const videoPlaylists: VideoPlaylistModel[] = await VideoPlaylistModel.findAll({
+    include: [
+      {
+        model: AccountModel.unscoped(),
+        required: true,
+        include: [
+          {
+            model: ActorModel.unscoped(),
+            where: {
+              serverId: null
+            },
+            required: true
+          }
+        ]
+      }
+    ]
+  })
+  for (const playlist of videoPlaylists) {
+    console.log('Updating video playlist ' + playlist.url)
+
+    playlist.url = getLocalVideoPlaylistActivityPubUrl(playlist)
+    await playlist.save()
+
+    const elements: VideoPlaylistElementModel[] = await VideoPlaylistElementModel.findAll({
+      where: {
+        videoPlaylistId: playlist.id
+      }
+    })
+
+    for (const element of elements) {
+      console.log('Updating video playlist element ' + element.url)
+
+      element.url = getLocalVideoPlaylistElementActivityPubUrl(playlist, element)
+      await element.save()
+    }
+  }
+
+  console.log('Updating video and torrent files.')
+
+  const ids = await VideoModel.listLocalIds()
+  for (const id of ids) {
+    const video = await VideoModel.loadFull(id)
+
+    console.log('Updating video ' + video.uuid)
+
+    video.url = getLocalVideoActivityPubUrl(video)
+    await video.save()
+
+    for (const file of video.VideoFiles) {
+      console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid)
+      await updateTorrentMetadata(video, file)
+
+      await file.save()
+    }
+
+    const playlist = video.getHLSPlaylist()
+    for (const file of (playlist?.VideoFiles || [])) {
+      console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid)
+
+      await updateTorrentMetadata(playlist, file)
+
+      await file.save()
+    }
+  }
+}

server/scripts/update-object-storage-url.ts

@@ -0,0 +1,102 @@
+/* eslint-disable max-len */
+import { InvalidArgumentError, createCommand } from '@commander-js/extra-typings'
+import { FileStorage } from '@peertube/peertube-models'
+import { escapeForRegex } from '@server/helpers/regexp.js'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { QueryTypes } from 'sequelize'
+import { askConfirmation, displayPeerTubeMustBeStoppedWarning } from './shared/common.js'
+
+const program = createCommand()
+  .description('Update PeerTube object file URLs after an object storage migration.')
+  .requiredOption('-f, --from <url>', 'Previous object storage base URL', parseUrl)
+  .requiredOption('-t, --to <url>', 'New object storage base URL', parseUrl)
+  .parse(process.argv)
+
+const options = program.opts()
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  displayPeerTubeMustBeStoppedWarning()
+
+  const fromRegexp = `^${escapeForRegex(options.from)}`
+  const to = options.to
+
+  const replacements = { fromRegexp, to, storage: FileStorage.OBJECT_STORAGE }
+
+  // Candidates
+  {
+    const queries = [
+      `SELECT COUNT(*) AS "c", 'videoFile->fileUrl: ' || COUNT(*) AS "t" FROM "videoFile" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->playlistUrl: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "playlistUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoStreamingPlaylist->segmentsSha256Url: ' || COUNT(*) AS "t" FROM "videoStreamingPlaylist" WHERE "segmentsSha256Url" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'userExport->fileUrl: ' || COUNT(*) AS "t" FROM "userExport" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoSource->fileUrl: ' || COUNT(*) AS "t" FROM "videoSource" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`,
+      `SELECT COUNT(*) AS "c", 'videoCaption->fileUrl: ' || COUNT(*) AS "t" FROM "videoCaption" WHERE "fileUrl" ~ :fromRegexp AND "storage" = :storage`
+    ]
+
+    let hasResults = false
+
+    console.log('Candidate URLs to update:')
+    for (const query of queries) {
+      const [ row ] = await sequelizeTypescript.query(query, { replacements, type: QueryTypes.SELECT as QueryTypes.SELECT })
+
+      if (row['c'] !== 0) hasResults = true
+
+      console.log(` ${row['t']}`)
+    }
+
+    console.log('\n')
+
+    if (!hasResults) {
+      console.log('No candidate URLs found, exiting.')
+      process.exit(0)
+    }
+  }
+
+  const res = await askUpdateConfirmation()
+  if (res !== true) {
+    console.log('Exiting without updating URLs.')
+    process.exit(0)
+  }
+
+  // Execute
+  {
+    const queries = [
+      `UPDATE "videoFile" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoStreamingPlaylist" SET "playlistUrl" = regexp_replace("playlistUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoStreamingPlaylist" SET "segmentsSha256Url" = regexp_replace("segmentsSha256Url", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "userExport" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoSource" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`,
+      `UPDATE "videoCaption" SET "fileUrl" = regexp_replace("fileUrl", :fromRegexp, :to) WHERE "storage" = :storage`
+    ]
+
+    for (const query of queries) {
+      await sequelizeTypescript.query(query, { replacements })
+    }
+
+    console.log('URLs updated.')
+  }
+}
+
+function parseUrl (value: string) {
+  if (!value || /^https?:\/\//.test(value) !== true) {
+    throw new InvalidArgumentError('Must be a valid URL (starting with http:// or https://).')
+  }
+
+  return value
+}
+
+async function askUpdateConfirmation () {
+  return askConfirmation(
+    'These URLs can be updated, but please check your backups first (bugs happen). ' +
+    'Can we update these URLs?'
+  )
+}

server/scripts/upgrade.sh

@@ -0,0 +1,109 @@
+#!/bin/sh
+
+set -eu
+
+PEERTUBE_PATH=${1:-/var/www/peertube}
+
+if [ ! -e "$PEERTUBE_PATH" ]; then
+  echo "Error - path \"$PEERTUBE_PATH\" wasn't found"
+  echo ""
+  echo "If peertube was installed in another path, you can specify it with"
+  echo "    ./upgrade.sh <PATH>"
+  exit 1
+fi
+
+if [ ! -e "$PEERTUBE_PATH/versions" -o ! -e "$PEERTUBE_PATH/config/production.yaml" ]; then
+  echo "Error - Couldn't find peertube installation in \"$PEERTUBE_PATH\""
+  echo ""
+  echo "If peertube was installed in another path, you can specify it with"
+  echo "    ./upgrade.sh <PATH>"
+  exit 1
+fi
+
+if [ -x "$(command -v awk)" ] && [ -x "$(command -v sed)" ]; then
+    REMAINING=$(df -k $PEERTUBE_PATH | awk '{ print $4}' | sed -n 2p)
+    THREE_GB=$((3 * 1024 * 1024))
+
+    if [ "$REMAINING" -lt "$THREE_GB" ]; then
+      echo "Error - not enough free space for upgrading"
+      echo ""
+      echo "Make sure you have at least 3 GB of free space in $PEERTUBE_PATH"
+      exit 1
+    fi
+fi
+
+# Backup database
+if [ -x "$(command -v pg_dump)" ]; then
+  mkdir -p $PEERTUBE_PATH/backup
+
+  SQL_BACKUP_PATH="$PEERTUBE_PATH/backup/sql-peertube_prod-$(date +"%Y%m%d-%H%M").bak"
+
+  echo "Backing up PostgreSQL database in $SQL_BACKUP_PATH"
+
+  DB_USER=$(node -e "console.log(require('js-yaml').load(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['username'])")
+  DB_PASS=$(node -e "console.log(require('js-yaml').load(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['password'])")
+  DB_HOST=$(node -e "console.log(require('js-yaml').load(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['hostname'])")
+  DB_PORT=$(node -e "console.log(require('js-yaml').load(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['port'])")
+  DB_SUFFIX=$(node -e "console.log(require('js-yaml').load(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['suffix'])")
+  DB_NAME=$(node -e "console.log(require('js-yaml').load(fs.readFileSync('$PEERTUBE_PATH/config/production.yaml', 'utf8'))['database']['name'] || '')")
+
+  PGPASSWORD=$DB_PASS pg_dump -U $DB_USER -p $DB_PORT -h $DB_HOST -F c "${DB_NAME:-peertube${DB_SUFFIX}}" -f "$SQL_BACKUP_PATH"
+else
+  echo "pg_dump not found. Cannot make a SQL backup!"
+fi
+
+# If there is a pre-release, give the user a choice which one to install.
+RELEASE_VERSION=$(curl -s https://api.github.com/repos/chocobozzz/peertube/releases/latest | grep tag_name | cut -d '"' -f 4)
+PRE_RELEASE_VERSION=$(curl -s https://api.github.com/repos/chocobozzz/peertube/releases | grep tag_name | head -1 | cut -d '"' -f 4)
+
+if [ "$RELEASE_VERSION" != "$PRE_RELEASE_VERSION" ]; then
+  echo -e "Which version do you want to install?\n[1] $RELEASE_VERSION (stable) \n[2] $PRE_RELEASE_VERSION (pre-release)"
+  read choice
+  case $choice in
+      [1]* ) VERSION="$RELEASE_VERSION";;
+      [2]* ) VERSION="$PRE_RELEASE_VERSION";;
+      * ) exit;
+  esac
+else
+  VERSION="$RELEASE_VERSION"
+fi
+
+echo "Installing Peertube version $VERSION"
+wget -q "https://github.com/Chocobozzz/PeerTube/releases/download/${VERSION}/peertube-${VERSION}.zip" -O "$PEERTUBE_PATH/versions/peertube-${VERSION}.zip"
+cd $PEERTUBE_PATH/versions
+unzip -o "peertube-${VERSION}.zip"
+rm -f "peertube-${VERSION}.zip"
+
+RELEASE_PAGE_URL="https://github.com/Chocobozzz/PeerTube/releases/tag/${VERSION}"
+LATEST_VERSION_DIRECTORY="$PEERTUBE_PATH/versions/peertube-${VERSION}"
+cd "$LATEST_VERSION_DIRECTORY"
+
+npm run install-node-dependencies -- --production
+
+OLD_VERSION_DIRECTORY=$(readlink "$PEERTUBE_PATH/peertube-latest")
+
+# Switch to latest code version
+rm -rf $PEERTUBE_PATH/peertube-latest
+ln -s "$LATEST_VERSION_DIRECTORY" $PEERTUBE_PATH/peertube-latest
+cp $PEERTUBE_PATH/peertube-latest/config/default.yaml $PEERTUBE_PATH/config/default.yaml
+
+echo ""
+echo "=========================================================="
+echo ""
+
+if [ -x "$(command -v git)" ]; then
+  cd $PEERTUBE_PATH
+
+  git merge-file -p config/production.yaml "$OLD_VERSION_DIRECTORY/config/production.yaml.example" "peertube-latest/config/production.yaml.example" | tee "config/production.yaml.new" > /dev/null
+  echo "$PEERTUBE_PATH/config/production.yaml.new generated"
+  echo "You can review it and replace your existing production.yaml configuration"
+else
+  echo "git command not found: unable to generate config/production.yaml.new configuration file based on your existing production.yaml configuration"
+fi
+
+echo ""
+echo "=========================================================="
+echo ""
+echo "Please read the IMPORTANT NOTES on $RELEASE_PAGE_URL"
+echo ""
+echo "Then restart PeerTube!"