Init commit
This commit is contained in:
240
server/core/lib/move-storage/move-to-file-system.ts
Normal file
240
server/core/lib/move-storage/move-to-file-system.ts
Normal file
@@ -0,0 +1,240 @@
|
||||
import { FileStorage, VideoStateType } from '@peertube/peertube-models'
|
||||
import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||
import {
|
||||
makeCaptionFileAvailable,
|
||||
makeHLSFileAvailable,
|
||||
makeOriginalFileAvailable,
|
||||
makeWebVideoFileAvailable,
|
||||
removeCaptionObjectStorage,
|
||||
removeHLSFileObjectStorageByFilename,
|
||||
removeOriginalFileObjectStorage,
|
||||
removeWebVideoObjectStorage
|
||||
} from '@server/lib/object-storage/index.js'
|
||||
import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||
import { updateHLSMasterOnCaptionChange, upsertCaptionPlaylistOnFS } from '@server/lib/video-captions.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedMoveToFileSystemState, moveToNextState } from '@server/lib/video-state.js'
|
||||
import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { join } from 'path'
|
||||
import { moveCaptionToStorage } from './shared/move-caption.js'
|
||||
import { moveVideoToStorage, onMoveVideoToStorageFailure } from './shared/move-video.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-file-system')
|
||||
|
||||
export async function moveVideoToFS (options: {
|
||||
videoUUID: string
|
||||
|
||||
moveVideoState?: {
|
||||
isNewVideo: boolean
|
||||
previousVideoState: VideoStateType
|
||||
}
|
||||
|
||||
loggerTags: LoggerTags['tags']
|
||||
}) {
|
||||
const { videoUUID, moveVideoState, loggerTags } = options
|
||||
|
||||
await moveVideoToStorage({
|
||||
videoUUID,
|
||||
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
|
||||
|
||||
targetStorage: FileStorage.FILE_SYSTEM,
|
||||
|
||||
moveWebVideoFiles,
|
||||
moveHLSFiles,
|
||||
moveVideoSourceFile,
|
||||
moveCaptionFiles
|
||||
})
|
||||
|
||||
if (options.moveVideoState) {
|
||||
const video = await VideoModel.load(videoUUID)
|
||||
|
||||
await moveToNextState({ video, ...moveVideoState })
|
||||
}
|
||||
}
|
||||
|
||||
export function moveCaptionToFS (options: {
|
||||
captionId: number
|
||||
loggerTags: LoggerTags['tags']
|
||||
}) {
|
||||
const { captionId, loggerTags } = options
|
||||
|
||||
return moveCaptionToStorage({
|
||||
captionId,
|
||||
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
|
||||
moveCaptionFiles
|
||||
})
|
||||
}
|
||||
|
||||
export async function onMoveVideoToFSFailure (options: {
|
||||
videoUUID: string
|
||||
loggerTags: LoggerTags['tags']
|
||||
err: Error
|
||||
}) {
|
||||
const { videoUUID, err, loggerTags } = options
|
||||
|
||||
await onMoveVideoToStorageFailure({
|
||||
videoUUID,
|
||||
err,
|
||||
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
|
||||
moveToFailedState: moveToFailedMoveToFileSystemState
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveVideoSourceFile (source: MVideoSource) {
|
||||
if (source.storage === FileStorage.FILE_SYSTEM) return
|
||||
|
||||
await makeOriginalFileAvailable(
|
||||
source.keptOriginalFilename,
|
||||
VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
|
||||
)
|
||||
|
||||
const oldFileUrl = source.fileUrl
|
||||
|
||||
source.fileUrl = null
|
||||
source.storage = FileStorage.FILE_SYSTEM
|
||||
await source.save()
|
||||
|
||||
logger.debug('Removing original video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||
|
||||
await removeOriginalFileObjectStorage(source)
|
||||
}
|
||||
|
||||
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
for (const file of video.VideoFiles) {
|
||||
if (file.storage === FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
await makeWebVideoFileAvailable(file.filename, VideoPathManager.Instance.getFSVideoFileOutputPath(video, file))
|
||||
await onVideoFileMoved({
|
||||
videoOrPlaylist: video,
|
||||
file,
|
||||
objetStorageRemover: () => removeWebVideoObjectStorage(file)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
for (const playlist of video.VideoStreamingPlaylists) {
|
||||
const playlistWithVideo = playlist.withVideo(video)
|
||||
|
||||
let updatedFile = false
|
||||
|
||||
for (const file of playlist.VideoFiles) {
|
||||
if (file.storage === FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
updatedFile = true
|
||||
|
||||
// Resolution playlist
|
||||
const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
|
||||
await makeHLSFileAvailable(playlistWithVideo, playlistFilename, join(getHLSDirectory(video), playlistFilename))
|
||||
await makeHLSFileAvailable(playlistWithVideo, file.filename, join(getHLSDirectory(video), file.filename))
|
||||
|
||||
await onVideoFileMoved({
|
||||
videoOrPlaylist: playlistWithVideo,
|
||||
file,
|
||||
objetStorageRemover: async () => {
|
||||
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlistFilename)
|
||||
await removeHLSFileObjectStorageByFilename(playlistWithVideo, file.filename)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if (playlistWithVideo.storage !== FileStorage.FILE_SYSTEM) {
|
||||
await makeHLSFileAvailable(playlistWithVideo, playlist.playlistFilename, join(getHLSDirectory(video), playlist.playlistFilename))
|
||||
|
||||
await makeHLSFileAvailable(
|
||||
playlistWithVideo,
|
||||
playlist.segmentsSha256Filename,
|
||||
join(getHLSDirectory(video), playlist.segmentsSha256Filename)
|
||||
)
|
||||
|
||||
playlist.playlistUrl = null
|
||||
playlist.segmentsSha256Url = null
|
||||
playlist.storage = FileStorage.FILE_SYSTEM
|
||||
|
||||
await playlist.save()
|
||||
|
||||
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlist.playlistFilename)
|
||||
await removeHLSFileObjectStorageByFilename(playlistWithVideo, playlist.segmentsSha256Filename)
|
||||
}
|
||||
|
||||
if (updatedFile === true) {
|
||||
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
||||
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
||||
|
||||
await playlist.save()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function onVideoFileMoved (options: {
|
||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||
file: MVideoFile
|
||||
objetStorageRemover: () => Promise<any>
|
||||
}) {
|
||||
const { videoOrPlaylist, file, objetStorageRemover } = options
|
||||
|
||||
const oldFileUrl = file.fileUrl
|
||||
|
||||
file.fileUrl = null
|
||||
file.storage = FileStorage.FILE_SYSTEM
|
||||
|
||||
await updateTorrentMetadata(videoOrPlaylist, file)
|
||||
await file.save()
|
||||
|
||||
logger.debug('Removing web video file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||
await objetStorageRemover()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
|
||||
let hlsUpdated = false
|
||||
|
||||
for (const caption of captions) {
|
||||
if (caption.storage === FileStorage.OBJECT_STORAGE) {
|
||||
const oldFileUrl = caption.fileUrl
|
||||
|
||||
await makeCaptionFileAvailable(caption.filename, caption.getFSFilePath())
|
||||
|
||||
// Assign new values before building the m3u8 file
|
||||
caption.fileUrl = null
|
||||
caption.storage = FileStorage.FILE_SYSTEM
|
||||
|
||||
await caption.save()
|
||||
|
||||
logger.debug('Removing caption file %s because it\'s now on file system', oldFileUrl, lTagsBase())
|
||||
await removeCaptionObjectStorage(caption)
|
||||
}
|
||||
|
||||
if (hls && (!caption.m3u8Filename || caption.m3u8Url)) {
|
||||
hlsUpdated = true
|
||||
|
||||
const oldM3U8Url = caption.m3u8Url
|
||||
const oldM3U8Filename = caption.m3u8Filename
|
||||
|
||||
// Caption link has been updated, so we must also update the HLS caption playlist
|
||||
caption.m3u8Filename = await upsertCaptionPlaylistOnFS(caption, hls.Video)
|
||||
caption.m3u8Url = null
|
||||
|
||||
await caption.save()
|
||||
|
||||
if (oldM3U8Url) {
|
||||
logger.debug(`Removing video caption playlist file ${oldM3U8Url} because it's now on file system`, lTagsBase())
|
||||
|
||||
await removeHLSFileObjectStorageByFilename(hls, oldM3U8Filename)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hlsUpdated) {
|
||||
await updateHLSMasterOnCaptionChange(hls.Video, hls)
|
||||
}
|
||||
}
|
||||
235
server/core/lib/move-storage/move-to-object-storage.ts
Normal file
235
server/core/lib/move-storage/move-to-object-storage.ts
Normal file
@@ -0,0 +1,235 @@
|
||||
import { FileStorage, VideoStateType } from '@peertube/peertube-models'
|
||||
import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js'
|
||||
import { buildCaptionM3U8Content } from '@server/lib/hls.js'
|
||||
import {
|
||||
storeHLSFileFromContent,
|
||||
storeHLSFileFromFilename,
|
||||
storeOriginalVideoFile,
|
||||
storeVideoCaption,
|
||||
storeWebVideoFile
|
||||
} from '@server/lib/object-storage/index.js'
|
||||
import { getHLSDirectory, getHLSResolutionPlaylistFilename } from '@server/lib/paths.js'
|
||||
import { updateHLSMasterOnCaptionChange } from '@server/lib/video-captions.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js'
|
||||
import { updateTorrentMetadata } from '@server/lib/webtorrent.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MStreamingPlaylistVideo, MVideo, MVideoCaption, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { rmdir } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { federateVideoIfNeeded } from '../activitypub/videos/federate.js'
|
||||
import { moveCaptionToStorage } from './shared/move-caption.js'
|
||||
import { moveVideoToStorage, onMoveVideoToStorageFailure } from './shared/move-video.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('object-storage', 'move-object-storage')
|
||||
|
||||
export async function moveVideoToObjectStorage (options: {
|
||||
videoUUID: string
|
||||
|
||||
moveVideoState?: {
|
||||
isNewVideo: boolean
|
||||
previousVideoState: VideoStateType
|
||||
}
|
||||
|
||||
loggerTags: LoggerTags['tags']
|
||||
}) {
|
||||
const { videoUUID, moveVideoState, loggerTags } = options
|
||||
|
||||
await moveVideoToStorage({
|
||||
videoUUID,
|
||||
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
|
||||
|
||||
targetStorage: FileStorage.OBJECT_STORAGE,
|
||||
|
||||
moveWebVideoFiles,
|
||||
moveHLSFiles,
|
||||
moveVideoSourceFile,
|
||||
moveCaptionFiles
|
||||
})
|
||||
|
||||
if (options.moveVideoState) {
|
||||
await moveToNextState({ video: { uuid: videoUUID }, ...moveVideoState })
|
||||
} else {
|
||||
const videoFull = await VideoModel.loadFull(videoUUID)
|
||||
await federateVideoIfNeeded(videoFull, false, undefined)
|
||||
}
|
||||
}
|
||||
|
||||
export function moveCaptionToObjectStorage (options: {
|
||||
captionId: number
|
||||
loggerTags: LoggerTags['tags']
|
||||
}) {
|
||||
const { captionId, loggerTags } = options
|
||||
|
||||
return moveCaptionToStorage({
|
||||
captionId,
|
||||
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
|
||||
moveCaptionFiles
|
||||
})
|
||||
}
|
||||
|
||||
export async function onMoveVideoToObjectStorageFailure (options: {
|
||||
videoUUID: string
|
||||
loggerTags: LoggerTags['tags']
|
||||
err: Error
|
||||
}) {
|
||||
const { videoUUID, err, loggerTags } = options
|
||||
|
||||
await onMoveVideoToStorageFailure({
|
||||
videoUUID,
|
||||
err,
|
||||
loggerTags: [ ...lTagsBase().tags, ...loggerTags ],
|
||||
moveToFailedState: moveToFailedMoveToObjectStorageState
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveVideoSourceFile (source: MVideoSource) {
|
||||
if (source.storage !== FileStorage.FILE_SYSTEM) return
|
||||
|
||||
const sourcePath = VideoPathManager.Instance.getFSOriginalVideoFilePath(source.keptOriginalFilename)
|
||||
const fileUrl = await storeOriginalVideoFile(sourcePath, source.keptOriginalFilename)
|
||||
|
||||
source.storage = FileStorage.OBJECT_STORAGE
|
||||
source.fileUrl = fileUrl
|
||||
await source.save()
|
||||
|
||||
logger.debug('Removing original video file ' + sourcePath + ' because it\'s now on object storage', lTagsBase())
|
||||
|
||||
await remove(sourcePath)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveCaptionFiles (captions: MVideoCaption[], hls: MStreamingPlaylistVideo) {
|
||||
let hlsUpdated = false
|
||||
|
||||
for (const caption of captions) {
|
||||
if (caption.storage === FileStorage.FILE_SYSTEM) {
|
||||
const captionPath = caption.getFSFilePath()
|
||||
|
||||
// Assign new values before building the m3u8 file
|
||||
caption.fileUrl = await storeVideoCaption(captionPath, caption.filename)
|
||||
caption.storage = FileStorage.OBJECT_STORAGE
|
||||
|
||||
await caption.save()
|
||||
|
||||
logger.debug(`Removing video caption file ${captionPath} because it's now on object storage`, lTagsBase())
|
||||
await remove(captionPath)
|
||||
}
|
||||
|
||||
if (hls && (!caption.m3u8Filename || !caption.m3u8Url)) {
|
||||
hlsUpdated = true
|
||||
|
||||
const m3u8PathToRemove = caption.getFSM3U8Path(hls.Video)
|
||||
|
||||
// Caption file URL has been updated, so we must also update the HLS caption playlist
|
||||
const content = buildCaptionM3U8Content({ video: hls.Video, caption })
|
||||
|
||||
caption.m3u8Filename = VideoCaptionModel.generateM3U8Filename(caption.filename)
|
||||
caption.m3u8Url = await storeHLSFileFromContent({
|
||||
playlist: hls,
|
||||
pathOrFilename: caption.m3u8Filename,
|
||||
content
|
||||
})
|
||||
|
||||
await caption.save()
|
||||
|
||||
if (m3u8PathToRemove) {
|
||||
logger.debug(`Removing video caption playlist file ${m3u8PathToRemove} because it's now on object storage`, lTagsBase())
|
||||
await remove(m3u8PathToRemove)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (hlsUpdated) {
|
||||
await updateHLSMasterOnCaptionChange(hls.Video, hls)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function moveWebVideoFiles (video: MVideoWithAllFiles) {
|
||||
for (const file of video.VideoFiles) {
|
||||
if (file.storage !== FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
const fileUrl = await storeWebVideoFile(video, file)
|
||||
|
||||
const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file)
|
||||
await onVideoFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath })
|
||||
}
|
||||
}
|
||||
|
||||
async function moveHLSFiles (video: MVideoWithAllFiles) {
|
||||
for (const playlist of video.VideoStreamingPlaylists) {
|
||||
const playlistWithVideo = playlist.withVideo(video)
|
||||
|
||||
let updatedFile = false
|
||||
|
||||
for (const file of playlist.VideoFiles) {
|
||||
if (file.storage !== FileStorage.FILE_SYSTEM) continue
|
||||
|
||||
updatedFile = true
|
||||
|
||||
// Resolution playlist
|
||||
const playlistFilename = getHLSResolutionPlaylistFilename(file.filename)
|
||||
await storeHLSFileFromFilename(playlistWithVideo, playlistFilename)
|
||||
|
||||
// Resolution fragmented file
|
||||
const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename)
|
||||
|
||||
const oldPath = join(getHLSDirectory(video), file.filename)
|
||||
|
||||
await onVideoFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath })
|
||||
|
||||
await remove(join(getHLSDirectory(video), playlistFilename))
|
||||
}
|
||||
|
||||
if (playlistWithVideo.storage === FileStorage.FILE_SYSTEM) {
|
||||
playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename)
|
||||
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename)
|
||||
playlist.storage = FileStorage.OBJECT_STORAGE
|
||||
|
||||
await playlist.save()
|
||||
|
||||
await remove(join(getHLSDirectory(video), playlist.playlistFilename))
|
||||
await remove(join(getHLSDirectory(video), playlist.segmentsSha256Filename))
|
||||
}
|
||||
|
||||
if (updatedFile === true) {
|
||||
playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles)
|
||||
playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION
|
||||
|
||||
await playlist.save()
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await rmdir(getHLSDirectory(video))
|
||||
} catch {
|
||||
// Nothing to do, directory may be not empty if there is a transcoding in progress
|
||||
}
|
||||
}
|
||||
|
||||
async function onVideoFileMoved (options: {
|
||||
videoOrPlaylist: MVideo | MStreamingPlaylistVideo
|
||||
file: MVideoFile
|
||||
fileUrl: string
|
||||
oldPath: string
|
||||
}) {
|
||||
const { videoOrPlaylist, file, fileUrl, oldPath } = options
|
||||
|
||||
file.fileUrl = fileUrl
|
||||
file.storage = FileStorage.OBJECT_STORAGE
|
||||
|
||||
await updateTorrentMetadata(videoOrPlaylist, file)
|
||||
await file.save()
|
||||
|
||||
logger.debug('Removing %s because it\'s now on object storage', oldPath, lTagsBase())
|
||||
await remove(oldPath)
|
||||
}
|
||||
49
server/core/lib/move-storage/shared/move-caption.ts
Normal file
49
server/core/lib/move-storage/shared/move-caption.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
|
||||
import { logger, LoggerTags, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { sequelizeTypescript } from '@server/initializers/database.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/federate.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MStreamingPlaylistVideoUUID, MVideoCaption } from '@server/types/models/index.js'
|
||||
|
||||
export async function moveCaptionToStorage (options: {
|
||||
captionId: number
|
||||
loggerTags: LoggerTags['tags']
|
||||
|
||||
moveCaptionFiles: (captions: MVideoCaption[], hls: MStreamingPlaylistVideoUUID) => Promise<void>
|
||||
}) {
|
||||
const {
|
||||
loggerTags,
|
||||
captionId,
|
||||
moveCaptionFiles
|
||||
} = options
|
||||
|
||||
const lTagsBase = loggerTagsFactory(...loggerTags)
|
||||
|
||||
const caption = await VideoCaptionModel.loadWithVideo(captionId)
|
||||
|
||||
if (!caption) {
|
||||
logger.info(`Can't process caption ${captionId}, caption does not exist anymore.`, lTagsBase())
|
||||
return
|
||||
}
|
||||
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(caption.Video.uuid)
|
||||
|
||||
const hls = await VideoStreamingPlaylistModel.loadHLSByVideoWithVideo(caption.videoId)
|
||||
|
||||
try {
|
||||
await moveCaptionFiles([ caption ], hls)
|
||||
|
||||
await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
const videoFull = await VideoModel.loadFull(caption.Video.id, t)
|
||||
|
||||
await federateVideoIfNeeded(videoFull, false, t)
|
||||
})
|
||||
})
|
||||
} finally {
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
135
server/core/lib/move-storage/shared/move-video.ts
Normal file
135
server/core/lib/move-storage/shared/move-video.ts
Normal file
@@ -0,0 +1,135 @@
|
||||
import { FileStorage, FileStorageType } from '@peertube/peertube-models'
|
||||
import { LoggerTags, logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MStreamingPlaylistVideoUUID, MVideo, MVideoCaption, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideoSource } from '@server/types/models/video/video-source.js'
|
||||
|
||||
export async function moveVideoToStorage (options: {
|
||||
videoUUID: string
|
||||
loggerTags: LoggerTags['tags']
|
||||
|
||||
targetStorage: FileStorageType
|
||||
|
||||
moveWebVideoFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||
moveHLSFiles: (video: MVideoWithAllFiles) => Promise<void>
|
||||
moveVideoSourceFile: (source: MVideoSource) => Promise<void>
|
||||
moveCaptionFiles: (captions: MVideoCaption[], hls: MStreamingPlaylistVideoUUID) => Promise<void>
|
||||
}) {
|
||||
const {
|
||||
loggerTags,
|
||||
videoUUID,
|
||||
moveVideoSourceFile,
|
||||
moveHLSFiles,
|
||||
moveWebVideoFiles,
|
||||
moveCaptionFiles,
|
||||
targetStorage
|
||||
} = options
|
||||
|
||||
const lTagsBase = loggerTagsFactory(...loggerTags)
|
||||
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoUUID)
|
||||
|
||||
const video = await VideoModel.loadWithFiles(videoUUID)
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info(`Can't move video ${videoUUID}, video does not exist.`, lTagsBase(videoUUID))
|
||||
fileMutexReleaser()
|
||||
return undefined
|
||||
}
|
||||
|
||||
const lTags = lTagsBase(video.uuid, video.url)
|
||||
|
||||
try {
|
||||
const { source, captions, hls, webFiles } = await filterVideoResourcesToBeMoved(video, targetStorage)
|
||||
|
||||
if (captions.length !== 0) {
|
||||
logger.debug(`Moving ${captions.length} captions of ${video.uuid}.`, lTags)
|
||||
|
||||
const hls = video.getHLSPlaylist()
|
||||
await moveCaptionFiles(captions, hls)
|
||||
}
|
||||
|
||||
if (source) {
|
||||
logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
|
||||
|
||||
await moveVideoSourceFile(source)
|
||||
}
|
||||
|
||||
if (webFiles.length !== 0) {
|
||||
logger.debug(`Moving ${webFiles.length} web video files for video ${video.uuid}.`, lTags)
|
||||
|
||||
await moveWebVideoFiles(video)
|
||||
}
|
||||
|
||||
if (hls) {
|
||||
logger.debug(`Moving HLS playlist of ${video.uuid}.`, lTags)
|
||||
|
||||
await moveHLSFiles(video)
|
||||
}
|
||||
|
||||
const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove')
|
||||
|
||||
logger.info(`Moved video ${video.uuid}. Remaining pending move: ${pendingMove}.`, lTags)
|
||||
} finally { // Error handling is managed by the job queue
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
export async function onMoveVideoToStorageFailure (options: {
|
||||
videoUUID: string
|
||||
err: any
|
||||
loggerTags: LoggerTags['tags']
|
||||
moveToFailedState: (video: MVideoWithAllFiles) => Promise<void>
|
||||
}) {
|
||||
const { videoUUID, err, loggerTags, moveToFailedState } = options
|
||||
|
||||
const video = await VideoModel.loadWithFiles(videoUUID)
|
||||
if (!video) return
|
||||
|
||||
logger.error(`Cannot move video ${video.url} storage.`, { err, tags: loggerTags })
|
||||
|
||||
await moveToFailedState(video)
|
||||
await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove')
|
||||
}
|
||||
|
||||
export async function filterVideoResourcesToBeMoved (videoArg: MVideo, targetStorage: FileStorageType) {
|
||||
const video = await VideoModel.loadFull(videoArg.id)
|
||||
const captions = await VideoCaptionModel.listVideoCaptions(video.id)
|
||||
const source = await VideoSourceModel.loadLatest(video.id)
|
||||
|
||||
const hls = video.getHLSPlaylist()
|
||||
|
||||
const moveHLS = hls && (hls.storage !== targetStorage || hls.VideoFiles.some(f => f.storage !== targetStorage))
|
||||
|
||||
return {
|
||||
source: source?.keptOriginalFilename && source.storage !== targetStorage
|
||||
? source
|
||||
: undefined,
|
||||
|
||||
hls: moveHLS
|
||||
? hls
|
||||
: undefined,
|
||||
|
||||
webFiles: video.VideoFiles.filter(f => f.storage !== targetStorage),
|
||||
captions: captions.filter(c => {
|
||||
if (c.storage !== targetStorage) return true
|
||||
|
||||
if (hls) {
|
||||
if (targetStorage === FileStorage.OBJECT_STORAGE) return !c.m3u8Filename || !c.m3u8Url
|
||||
else if (targetStorage === FileStorage.FILE_SYSTEM) return !c.m3u8Filename || c.m3u8Url
|
||||
}
|
||||
|
||||
return false
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export async function hasVideoResourcesToBeMoved (video: MVideo, targetStorage: FileStorageType) {
|
||||
const { captions, hls, source, webFiles } = await filterVideoResourcesToBeMoved(video, targetStorage)
|
||||
|
||||
return captions.length !== 0 || !!hls || !!source || webFiles.length !== 0
|
||||
}
|
||||
Reference in New Issue
Block a user