Init commit

server/scripts/migrations/peertube-4.0.ts

@@ -0,0 +1,109 @@
+import Bluebird from 'bluebird'
+import { move } from 'fs-extra/esm'
+import { readFile, writeFile } from 'fs/promises'
+import { join } from 'path'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import {
+  generateHLSMasterPlaylistFilename,
+  generateHlsSha256SegmentsFilename,
+  getHLSResolutionPlaylistFilename
+} from '@server/lib/paths.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
+import { VideoModel } from '@server/models/video/video.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Migrate old HLS paths to new format.')
+
+  await initDatabaseModels(true)
+
+  JobQueue.Instance.init()
+
+  const ids = await VideoModel.listLocalIds()
+
+  await Bluebird.map(ids, async id => {
+    try {
+      await processVideo(id)
+    } catch (err) {
+      console.error('Cannot process video %s.', { err })
+    }
+  }, { concurrency: 5 })
+
+  console.log('Migration finished!')
+}
+
+async function processVideo (videoId: number) {
+  const video = await VideoModel.loadWithFiles(videoId)
+
+  const hls = video.getHLSPlaylist()
+  if (video.isLive || !hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) {
+    return
+  }
+
+  console.log(`Renaming HLS playlist files of video ${video.name}.`)
+
+  const playlist = await VideoStreamingPlaylistModel.loadHLSByVideo(video.id)
+  const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video)
+
+  const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename)
+  let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8')
+
+  for (const videoFile of hls.VideoFiles) {
+    const srcName = `${videoFile.resolution}.m3u8`
+    const dstName = getHLSResolutionPlaylistFilename(videoFile.filename)
+
+    const src = join(hlsDirPath, srcName)
+    const dst = join(hlsDirPath, dstName)
+
+    try {
+      await move(src, dst)
+
+      masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName)
+    } catch (err) {
+      console.error('Cannot move video file %s to %s.', src, dst, err)
+    }
+  }
+
+  await writeFile(masterPlaylistPath, masterPlaylistContent)
+
+  if (playlist.segmentsSha256Filename === 'segments-sha256.json') {
+    try {
+      const newName = generateHlsSha256SegmentsFilename(video.isLive)
+
+      const dst = join(hlsDirPath, newName)
+      await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst)
+      playlist.segmentsSha256Filename = newName
+    } catch (err) {
+      console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err)
+    }
+  }
+
+  if (playlist.playlistFilename === 'master.m3u8') {
+    try {
+      const newName = generateHLSMasterPlaylistFilename(video.isLive)
+
+      const dst = join(hlsDirPath, newName)
+      await move(join(hlsDirPath, playlist.playlistFilename), dst)
+      playlist.playlistFilename = newName
+    } catch (err) {
+      console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err)
+    }
+  }
+
+  // Everything worked, we can save the playlist now
+  await playlist.save()
+
+  const allVideo = await VideoModel.loadFull(video.id)
+  await federateVideoIfNeeded(allVideo, false)
+
+  console.log(`Successfully moved HLS files of ${video.name}.`)
+}

server/scripts/migrations/peertube-4.2.ts

@@ -0,0 +1,122 @@
+import { maxBy, minBy } from '@peertube/peertube-core-utils'
+import { ActorImageType } from '@peertube/peertube-models'
+import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils'
+import { getImageSize, processImage } from '@server/helpers/image-utils.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants.js'
+import { initDatabaseModels } from '@server/initializers/database.js'
+import { updateActorImages } from '@server/lib/activitypub/actors/index.js'
+import { sendUpdateActor } from '@server/lib/activitypub/send/index.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import { AccountModel } from '@server/models/account/account.js'
+import { ActorModel } from '@server/models/actor/actor.js'
+import { VideoChannelModel } from '@server/models/video/video-channel.js'
+import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models/index.js'
+import { join } from 'path'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Generate avatar miniatures from existing avatars.')
+
+  await initDatabaseModels(true)
+  JobQueue.Instance.init()
+
+  const accounts: AccountModel[] = await AccountModel.findAll({
+    include: [
+      {
+        model: ActorModel,
+        required: true,
+        where: {
+          serverId: null
+        }
+      },
+      {
+        model: VideoChannelModel,
+        include: [
+          {
+            model: AccountModel
+          }
+        ]
+      }
+    ]
+  })
+
+  for (const account of accounts) {
+    try {
+      await fillAvatarSizeIfNeeded(account)
+      await generateSmallerAvatarIfNeeded(account)
+    } catch (err) {
+      console.error(`Cannot process account avatar ${account.name}`, err)
+    }
+
+    for (const videoChannel of account.VideoChannels) {
+      try {
+        await fillAvatarSizeIfNeeded(videoChannel)
+        await generateSmallerAvatarIfNeeded(videoChannel)
+      } catch (err) {
+        console.error(`Cannot process channel avatar ${videoChannel.name}`, err)
+      }
+    }
+  }
+
+  console.log('Generation finished!')
+}
+
+async function fillAvatarSizeIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) {
+  const avatars = accountOrChannel.Actor.Avatars
+
+  for (const avatar of avatars) {
+    if (avatar.width && avatar.height) continue
+
+    console.log('Filling size of avatars of %s.', accountOrChannel.name)
+
+    const { width, height } = await getImageSize(join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, avatar.filename))
+    avatar.width = width
+    avatar.height = height
+
+    await avatar.save()
+  }
+}
+
+async function generateSmallerAvatarIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) {
+  const avatars = accountOrChannel.Actor.Avatars
+  if (avatars.length !== 1) {
+    return
+  }
+
+  console.log(`Processing ${accountOrChannel.name}.`)
+
+  await generateSmallerAvatar(accountOrChannel.Actor)
+  accountOrChannel.Actor = Object.assign(accountOrChannel.Actor, { Server: null })
+
+  return sendUpdateActor(accountOrChannel, undefined)
+}
+
+async function generateSmallerAvatar (actor: MActorDefault) {
+  const bigAvatar = maxBy(actor.Avatars, 'width')
+
+  const imageSize = minBy(ACTOR_IMAGES_SIZE[ActorImageType.AVATAR], 'width')
+  const sourceFilename = bigAvatar.filename
+
+  const newImageName = buildUUID() + getLowercaseExtension(sourceFilename)
+  const source = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, sourceFilename)
+  const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, newImageName)
+
+  await processImage({ path: source, destination, newSize: imageSize, keepOriginal: true })
+
+  const actorImageInfo = {
+    name: newImageName,
+    fileUrl: null,
+    height: imageSize.height,
+    width: imageSize.width,
+    onDisk: true
+  }
+
+  await updateActorImages(actor, ActorImageType.AVATAR, [ actorImageInfo ], undefined)
+}

server/scripts/migrations/peertube-5.0.ts

@@ -0,0 +1,71 @@
+import { ensureDir } from 'fs-extra/esm'
+import { Op } from 'sequelize'
+import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
+import { DIRECTORIES } from '@server/initializers/constants.js'
+import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideoFullLight } from '@server/types/models/index.js'
+import { VideoPrivacy } from '@peertube/peertube-models'
+import { initDatabaseModels } from '@server/initializers/database.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  console.log('Moving private video files in dedicated folders.')
+
+  await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)
+  await ensureDir(DIRECTORIES.WEB_VIDEOS.PRIVATE)
+
+  await initDatabaseModels(true)
+
+  const videos = await VideoModel.unscoped().findAll({
+    attributes: [ 'uuid' ],
+    where: {
+      privacy: {
+        [Op.in]: [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]
+      }
+    }
+  })
+
+  for (const { uuid } of videos) {
+    try {
+      console.log('Moving files of video %s.', uuid)
+
+      const video = await VideoModel.loadFull(uuid)
+
+      try {
+        await moveFilesIfPrivacyChanged(video, VideoPrivacy.PUBLIC)
+      } catch (err) {
+        console.error('Cannot move files of video %s.', uuid, err)
+      }
+
+      try {
+        await updateTorrents(video)
+      } catch (err) {
+        console.error('Cannot regenerate torrents of video %s.', uuid, err)
+      }
+    } catch (err) {
+      console.error('Cannot process video %s.', uuid, err)
+    }
+  }
+}
+
+async function updateTorrents (video: MVideoFullLight) {
+  for (const file of video.VideoFiles) {
+    await updateTorrentMetadata(video, file)
+
+    await file.save()
+  }
+
+  const playlist = video.getHLSPlaylist()
+  for (const file of (playlist?.VideoFiles || [])) {
+    await updateTorrentMetadata(playlist, file)
+
+    await file.save()
+  }
+}

server/scripts/migrations/peertube-6.3.ts

@@ -0,0 +1,133 @@
+import { ffprobePromise, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
+import { VideoFileStream } from '@peertube/peertube-models'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { buildFileMetadata } from '@server/lib/video-file.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { VideoFileModel } from '@server/models/video/video-file.js'
+import { VideoModel } from '@server/models/video/video.js'
+import Bluebird from 'bluebird'
+import { pathExists } from 'fs-extra/esm'
+import { QueryTypes } from 'sequelize'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  {
+    console.log('## Updating "formatFlags" column for web videos in "videoFile" table in database ##\n')
+
+    const totalQuery = 'SELECT COUNT(*) AS "total" FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
+    const res = await sequelizeTypescript.query<{ total: string }>(totalQuery, { type: QueryTypes.SELECT as QueryTypes.SELECT })
+    const total = parseInt(res[0].total)
+
+    console.log(`Will update ${total.toLocaleString()} rows`)
+
+    if (total > 10000) {
+      console.log('Processing update in chunks because there are many rows to update...')
+
+      const chunkSize = 10000
+      let remaining = total
+
+      while (remaining > chunkSize) {
+        const before = new Date().getTime()
+
+        await sequelizeTypescript.query(
+          'UPDATE "videoFile" SET "formatFlags" = 1 WHERE id IN (' +
+            'SELECT id FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1 LIMIT ' + chunkSize +
+          ')'
+        )
+
+        remaining -= chunkSize
+
+        const ms = new Date().getTime() - before
+        console.log(`Processed ${chunkSize.toLocaleString()} rows in ${ms.toLocaleString()}ms. Remaining: ${remaining.toLocaleString()}`)
+      }
+    }
+
+    const query = 'UPDATE "videoFile" SET "formatFlags" = 1 WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
+    await sequelizeTypescript.query(query)
+
+    console.log('Rows updated!')
+  }
+
+  console.log('## Assigning metadata information to local video files ##\n')
+
+  const ids = await VideoModel.listLocalIds()
+
+  await Bluebird.map(ids, async id => {
+    try {
+      await processVideo(id)
+    } catch (err) {
+      console.error('Cannot process video ' + id, err)
+    }
+  }, { concurrency: 5 })
+
+  console.log('\n## Migration finished! ##')
+}
+
+async function processVideo (videoId: number) {
+  const video = await VideoModel.loadWithFiles(videoId)
+  if (video.isLive) return
+
+  const files = await Promise.all(video.getAllFiles().map(f => VideoFileModel.loadWithMetadata(f.id)))
+
+  if (!files.some(f => f.fps === -1 || !f.metadata || !f.streams || f.width === null || f.height === null)) return
+
+  console.log('Processing video "' + video.name + '"')
+
+  for (const file of files) {
+    if (!file.metadata || file.fps === -1) {
+      const fileWithVideoOrPlaylist = file.videoStreamingPlaylistId
+        ? file.withVideoOrPlaylist(video.getHLSPlaylist())
+        : file.withVideoOrPlaylist(video)
+
+      await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async path => {
+        if (!await pathExists(path)) {
+          console.error(
+            `Skipping processing file ${file.id} because ${path} does not exist on disk for video "${video.name}" (uuid: ${video.uuid})`
+          )
+          return
+        }
+
+        console.log(`Filling metadata field of video "${video.name}" - file ${file.id} because it is missing in database`)
+
+        const probe = await ffprobePromise(path)
+
+        file.metadata = await buildFileMetadata(path, probe)
+        file.fps = await getVideoStreamFPS(path, probe)
+
+        await file.save()
+      })
+    }
+
+    if (!file.metadata) continue
+
+    file.streams = VideoFileStream.NONE
+
+    const videoStream = file.metadata.streams.find(s => s.codec_type === 'video')
+
+    if (videoStream) {
+      file.streams |= VideoFileStream.VIDEO
+
+      file.width = videoStream.width
+      file.height = videoStream.height
+    } else {
+      file.width = 0
+      file.height = 0
+    }
+
+    if (file.metadata.streams.some(s => s.codec_type === 'audio')) {
+      file.streams |= VideoFileStream.AUDIO
+    }
+
+    await file.save()
+  }
+
+  console.log('Successfully processed video "' + video.name + '"')
+}

server/scripts/migrations/peertube-7.2.ts

@@ -0,0 +1,20 @@
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  console.log('Running SQL request to update comments count...')
+  {
+    const query = 'UPDATE "video" SET "comments" = (SELECT COUNT(*) FROM "videoComment" WHERE "videoComment"."videoId" = "video"."id")'
+    await sequelizeTypescript.query(query)
+  }
+
+  console.log('Done!')
+}

server/scripts/migrations/peertube-8.0.ts

@@ -0,0 +1,45 @@
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
+import { ApplicationModel } from '@server/models/application/application.js'
+import { QueryTypes } from 'sequelize'
+
+run()
+  .then(() => process.exit(0))
+  .catch(err => {
+    console.error(err)
+    process.exit(-1)
+  })
+
+async function run () {
+  await initDatabaseModels(true)
+
+  const application = await ApplicationModel.load()
+  if (application.migrationVersion < 960) {
+    console.error('This migration script must be run after v8.0 migration')
+    process.exit(-1)
+  }
+
+  const chunkSize = 500
+  let i = 1
+
+  console.log(`Running SQL request to delete unused actors, in chunks of maximum ${chunkSize} rows...`)
+
+  while (true) {
+    console.log('Processing chunk ' + i)
+
+    const query = `WITH cte AS (SELECT id FROM "actor" WHERE "accountId" IS NULL AND "videoChannelId" IS NULL LIMIT ${chunkSize}) ` +
+      `DELETE FROM "actor" WHERE id IN (SELECT "id" FROM cte) RETURNING id`
+
+    const deleted = await sequelizeTypescript.query(query, {
+      type: QueryTypes.BULKDELETE,
+      replacements: { limit: chunkSize }
+    })
+
+    if (deleted < chunkSize) {
+      break
+    }
+
+    i++
+  }
+
+  console.log('Done!')
+}