Init commit

This commit is contained in:
ShreejitPanchal
2026-05-19 19:56:02 +08:00
commit 6601501eff
4047 changed files with 2185372 additions and 0 deletions

View File

@@ -0,0 +1,240 @@
import { FileStorage, VideoStateType } from '@peertube/peertube-models'
import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
import {
makeCaptionFileAvailable,
makeHLSFileAvailable,
makeOriginalFileAvailable,
makeWebVideoFileAvailable,
removeCaptionObjectStorage,
removeHLSFileObjectStorageByFilename,
removeOriginalFileObjectStorage,
removeWebVideoObjectStorage
} from '@server/lib/object-storage/index.js'
import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
import { updateHLSMasterOnCaptionChange, upsertCaptionPlaylistOnFS } from '@server/lib/video-captions.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { join } from 'path'
import { moveCaptionToStorage } from './shared/move-caption.js'
import { moveVideoToStorage, onMoveVideoToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('move-file-system')
export async function moveVideoToFS (options: {
videoUUID: string
moveVideoState?: {
isNewVideo: boolean
previousVideoState: VideoStateType
}
loggerTags: LoggerTags['tags']
}) {
const { videoUUID, moveVideoState, loggerTags } = options
await moveVideoToStorage({
videoUUID,
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
targetStorage: FileStorage.FILE_SYSTEM,
moveWebVideoFiles,
moveHLSFiles,
moveVideoSourceFile,
moveCaptionFiles
})
if (options.moveVideoState) {
const video = await VideoModel.load(videoUUID)
await moveToNextState({ video, ...moveVideoState })
}
}
export function moveCaptionToFS (options: {
captionId: number
loggerTags: LoggerTags['tags']
}) {
const { captionId, loggerTags } = options
return moveCaptionToStorage({
captionId,
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
moveCaptionFiles
})
}
export async function onMoveVideoToFSFailure (options: {
videoUUID: string
loggerTags: LoggerTags['tags']
err: Error
}) {
const { videoUUID, err, loggerTags } = options
await onMoveVideoToStorageFailure({
videoUUID,
err,
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
moveToFailedState: moveToFailedMoveToFileSystemState
})
}
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------
async function moveVideoSourceFile (source: MVideoSource) {
if (source.storage === FileStorage.FILE_SYSTEM) return
await makeOriginalFileAvailable(
source.keptOriginalFilename,
VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
)
const oldFileUrl = source.fileUrl
source.fileUrl = null
source.storage = FileStorage.FILE_SYSTEM
await source.save()
logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
await removeOriginalFileObjectStorage(source)
}
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage === FileStorage.FILE_SYSTEM) continue
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
await onVideoFileMoved({
videoOrPlaylist: video,
file,
objetStorageRemover: () => removeWebVideoObjectStorage(file)
})
}
}
async function moveHLSFiles (video: MVideoWithAllFiles) {
for (const playlist of video.VideoStreamingPlaylists) {
const playlistWithVideo = playlist.withVideo(video)
let updatedFile = false
for (const file of playlist.VideoFiles) {
if (file.storage === FileStorage.FILE_SYSTEM) continue
updatedFile = true
// Resolution playlist
const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
await onVideoFileMoved({
videoOrPlaylist: playlistWithVideo,
file,
objetStorageRemover: async () => {
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
}
})
}
if (playlistWithVideo.storage !== FileStorage.FILE_SYSTEM) {
await makeHLSFileAvailable(playlistWithVideo, playlist.playlistFilename, join(getHLSDirectory(video), playlist.playlistFilename))
await makeHLSFileAvailable(
playlistWithVideo,
playlist.segmentsSha256Filename,
join(getHLSDirectory(video), playlist.segmentsSha256Filename)
)
playlist.playlistUrl = null
playlist.segmentsSha256Url = null
playlist.storage = FileStorage.FILE_SYSTEM
await playlist.save()
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlist.playlistFilename)
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlist.segmentsSha256Filename)
}
if (updatedFile === true) {
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
await playlist.save()
}
}
}
async function onVideoFileMoved (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
file: MVideoFile
objetStorageRemover: () => Promise<any>
}) {
const { videoOrPlaylist, file, objetStorageRemover } = options
const oldFileUrl = file.fileUrl
file.fileUrl = null
file.storage = FileStorage.FILE_SYSTEM
await updateTorrentMetadata(videoOrPlaylist, file)
await file.save()
logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
await objetStorageRemover()
}
// ---------------------------------------------------------------------------
async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
let hlsUpdated = false
for (const caption of captions) {
if (caption.storage === FileStorage.OBJECT_STORAGE) {
const oldFileUrl = caption.fileUrl
await makeCaptionFileAvailable(caption.filename, caption.getFSFilePath())
// Assign new values before building the m3u8 file
caption.fileUrl = null
caption.storage = FileStorage.FILE_SYSTEM
await caption.save()
logger.debug('Removing caption file %s because it\'s now on file system', oldFileUrl, lTagsBase())
await removeCaptionObjectStorage(caption)
}
if (hls && (!caption.m3u8Filename || caption.m3u8Url)) {
hlsUpdated = true
const oldM3U8Url = caption.m3u8Url
const oldM3U8Filename = caption.m3u8Filename
// Caption link has been updated, so we must also update the HLS caption playlist
caption.m3u8Filename = await upsertCaptionPlaylistOnFS(caption, hls.Video)
caption.m3u8Url = null
await caption.save()
if (oldM3U8Url) {
logger.debug(`Removing video caption playlist file ${oldM3U8Url} because it's now on file system`, lTagsBase())
await removeHLSFileObjectStorageByFilename(hls, oldM3U8Filename)
}
}
}
if (hlsUpdated) {
await updateHLSMasterOnCaptionChange(hls.Video, hls)
}
}

View File

@@ -0,0 +1,235 @@
import { FileStorage, VideoStateType } from '@peertube/peertube-models'
import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
import { buildCaptionM3U8Content } from '@server/lib/hls.js'
import {
storeHLSFileFromContent,
storeHLSFileFromFilename,
storeOriginalVideoFile,
storeVideoCaption,
storeWebVideoFile
} from '@server/lib/object-storage/index.js'
import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
import { updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { remove } from 'fs-extra/esm'
import { rmdir } from 'fs/promises'
import { join } from 'path'
import { federateVideoIfNeeded } from '../activitypub/videos/federate.js'
import { moveCaptionToStorage } from './shared/move-caption.js'
import { moveVideoToStorage, onMoveVideoToStorageFailure } from './shared/move-video.js'
const lTagsBase = loggerTagsFactory('object-storage', 'move-object-storage')
export async function moveVideoToObjectStorage (options: {
videoUUID: string
moveVideoState?: {
isNewVideo: boolean
previousVideoState: VideoStateType
}
loggerTags: LoggerTags['tags']
}) {
const { videoUUID, moveVideoState, loggerTags } = options
await moveVideoToStorage({
videoUUID,
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
targetStorage: FileStorage.OBJECT_STORAGE,
moveWebVideoFiles,
moveHLSFiles,
moveVideoSourceFile,
moveCaptionFiles
})
if (options.moveVideoState) {
await moveToNextState({ video: { uuid: videoUUID }, ...moveVideoState })
} else {
const videoFull = await VideoModel.loadFull(videoUUID)
await federateVideoIfNeeded(videoFull, false, undefined)
}
}
export function moveCaptionToObjectStorage (options: {
captionId: number
loggerTags: LoggerTags['tags']
}) {
const { captionId, loggerTags } = options
return moveCaptionToStorage({
captionId,
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
moveCaptionFiles
})
}
export async function onMoveVideoToObjectStorageFailure (options: {
videoUUID: string
loggerTags: LoggerTags['tags']
err: Error
}) {
const { videoUUID, err, loggerTags } = options
await onMoveVideoToStorageFailure({
videoUUID,
err,
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
moveToFailedState: moveToFailedMoveToObjectStorageState
})
}
// ---------------------------------------------------------------------------
async function moveVideoSourceFile (source: MVideoSource) {
if (source.storage !== FileStorage.FILE_SYSTEM) return
const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
source.storage = FileStorage.OBJECT_STORAGE
source.fileUrl = fileUrl
await source.save()
logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
await remove(sourcePath)
}
// ---------------------------------------------------------------------------
async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
let hlsUpdated = false
for (const caption of captions) {
if (caption.storage === FileStorage.FILE_SYSTEM) {
const captionPath = caption.getFSFilePath()
// Assign new values before building the m3u8 file
caption.fileUrl = await storeVideoCaption(captionPath, caption.filename)
caption.storage = FileStorage.OBJECT_STORAGE
await caption.save()
logger.debug(`Removing video caption file ${captionPath} because it's now on object storage`, lTagsBase())
await remove(captionPath)
}
if (hls && (!caption.m3u8Filename || !caption.m3u8Url)) {
hlsUpdated = true
const m3u8PathToRemove = caption.getFSM3U8Path(hls.Video)
// Caption file URL has been updated, so we must also update the HLS caption playlist
const content = buildCaptionM3U8Content({ video: hls.Video, caption })
caption.m3u8Filename = VideoCaptionModel.generateM3U8Filename(caption.filename)
caption.m3u8Url = await storeHLSFileFromContent({
playlist: hls,
pathOrFilename: caption.m3u8Filename,
content
})
await caption.save()
if (m3u8PathToRemove) {
logger.debug(`Removing video caption playlist file ${m3u8PathToRemove} because it's now on object storage`, lTagsBase())
await remove(m3u8PathToRemove)
}
}
}
if (hlsUpdated) {
await updateHLSMasterOnCaptionChange(hls.Video, hls)
}
}
// ---------------------------------------------------------------------------
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
for (const file of video.VideoFiles) {
if (file.storage !== FileStorage.FILE_SYSTEM) continue
const fileUrl = await storeWebVideoFile(video, file)
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
}
}
async function moveHLSFiles (video: MVideoWithAllFiles) {
for (const playlist of video.VideoStreamingPlaylists) {
const playlistWithVideo = playlist.withVideo(video)
let updatedFile = false
for (const file of playlist.VideoFiles) {
if (file.storage !== FileStorage.FILE_SYSTEM) continue
updatedFile = true
// Resolution playlist
const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
// Resolution fragmented file
const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
const oldPath = join(getHLSDirectory(video), file.filename)
await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
await remove(join(getHLSDirectory(video), playlistFilename))
}
if (playlistWithVideo.storage === FileStorage.FILE_SYSTEM) {
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
playlist.storage = FileStorage.OBJECT_STORAGE
await playlist.save()
await remove(join(getHLSDirectory(video), playlist.playlistFilename))
await remove(join(getHLSDirectory(video), playlist.segmentsSha256Filename))
}
if (updatedFile === true) {
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
await playlist.save()
}
}
try {
await rmdir(getHLSDirectory(video))
} catch {
// Nothing to do, directory may be not empty if there is a transcoding in progress
}
}
async function onVideoFileMoved (options: {
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
file: MVideoFile
fileUrl: string
oldPath: string
}) {
const { videoOrPlaylist, file, fileUrl, oldPath } = options
file.fileUrl = fileUrl
file.storage = FileStorage.OBJECT_STORAGE
await updateTorrentMetadata(videoOrPlaylist, file)
await file.save()
logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
await remove(oldPath)
}

View File

@@ -0,0 +1,49 @@
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/federate.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideoUUID, MVideoCaption } from '@server/types/models/index.js'
export async function moveCaptionToStorage (options: {
captionId: number
loggerTags: LoggerTags['tags']
moveCaptionFiles: (captions: MVideoCaption[], hls: MStreamingPlaylistVideoUUID) => Promise<void>
}) {
const {
loggerTags,
captionId,
moveCaptionFiles
} = options
const lTagsBase = loggerTagsFactory(...loggerTags)
const caption = await VideoCaptionModel.loadWithVideo(captionId)
if (!caption) {
logger.info(`Can't process caption ${captionId}, caption does not exist anymore.`, lTagsBase())
return
}
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(caption.Video.uuid)
const hls = await VideoStreamingPlaylistModel.loadHLSByVideoWithVideo(caption.videoId)
try {
await moveCaptionFiles([ caption ], hls)
await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async t => {
const videoFull = await VideoModel.loadFull(caption.Video.id, t)
await federateVideoIfNeeded(videoFull, false, t)
})
})
} finally {
fileMutexReleaser()
}
}

View File

@@ -0,0 +1,135 @@
import { FileStorage, FileStorageType } from '@peertube/peertube-models'
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { VideoModel } from '@server/models/video/video.js'
import { MStreamingPlaylistVideoUUID, MVideo, MVideoCaption, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
export async function moveVideoToStorage (options: {
videoUUID: string
loggerTags: LoggerTags['tags']
targetStorage: FileStorageType
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
moveCaptionFiles: (captions: MVideoCaption[], hls: MStreamingPlaylistVideoUUID) => Promise<void>
}) {
const {
loggerTags,
videoUUID,
moveVideoSourceFile,
moveHLSFiles,
moveWebVideoFiles,
moveCaptionFiles,
targetStorage
} = options
const lTagsBase = loggerTagsFactory(...loggerTags)
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoUUID)
const video = await VideoModel.loadWithFiles(videoUUID)
// No video, maybe deleted?
if (!video) {
logger.info(`Can't move video ${videoUUID}, video does not exist.`, lTagsBase(videoUUID))
fileMutexReleaser()
return undefined
}
const lTags = lTagsBase(video.uuid, video.url)
try {
const { source, captions, hls, webFiles } = await filterVideoResourcesToBeMoved(video, targetStorage)
if (captions.length !== 0) {
logger.debug(`Moving ${captions.length} captions of ${video.uuid}.`, lTags)
const hls = video.getHLSPlaylist()
await moveCaptionFiles(captions, hls)
}
if (source) {
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
await moveVideoSourceFile(source)
}
if (webFiles.length !== 0) {
logger.debug(`Moving ${webFiles.length} web video files for video ${video.uuid}.`, lTags)
await moveWebVideoFiles(video)
}
if (hls) {
logger.debug(`Moving HLS playlist of ${video.uuid}.`, lTags)
await moveHLSFiles(video)
}
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
logger.info(`Moved video ${video.uuid}. Remaining pending move: ${pendingMove}.`, lTags)
} finally { // Error handling is managed by the job queue
fileMutexReleaser()
}
}
export async function onMoveVideoToStorageFailure (options: {
videoUUID: string
err: any
loggerTags: LoggerTags['tags']
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
}) {
const { videoUUID, err, loggerTags, moveToFailedState } = options
const video = await VideoModel.loadWithFiles(videoUUID)
if (!video) return
logger.error(`Cannot move video ${video.url} storage.`, { err, tags: loggerTags })
await moveToFailedState(video)
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
}
export async function filterVideoResourcesToBeMoved (videoArg: MVideo, targetStorage: FileStorageType) {
const video = await VideoModel.loadFull(videoArg.id)
const captions = await VideoCaptionModel.listVideoCaptions(video.id)
const source = await VideoSourceModel.loadLatest(video.id)
const hls = video.getHLSPlaylist()
const moveHLS = hls && (hls.storage !== targetStorage || hls.VideoFiles.some(f => f.storage !== targetStorage))
return {
source: source?.keptOriginalFilename && source.storage !== targetStorage
? source
: undefined,
hls: moveHLS
? hls
: undefined,
webFiles: video.VideoFiles.filter(f => f.storage !== targetStorage),
captions: captions.filter(c => {
if (c.storage !== targetStorage) return true
if (hls) {
if (targetStorage === FileStorage.OBJECT_STORAGE) return !c.m3u8Filename || !c.m3u8Url
else if (targetStorage === FileStorage.FILE_SYSTEM) return !c.m3u8Filename || c.m3u8Url
}
return false
})
}
}
export async function hasVideoResourcesToBeMoved (video: MVideo, targetStorage: FileStorageType) {
const { captions, hls, source, webFiles } = await filterVideoResourcesToBeMoved(video, targetStorage)
return captions.length !== 0 || !!hls || !!source || webFiles.length !== 0
}