Init commit

This commit is contained in:
ShreejitPanchal
2026-05-19 19:56:02 +08:00
commit 6601501eff
4047 changed files with 2185372 additions and 0 deletions

View File

@@ -0,0 +1,202 @@
import Bluebird from 'bluebird'
import { Job } from 'bullmq'
import {
isAnnounceActivityValid,
isDislikeActivityValid,
isLikeActivityValid
} from '@server/helpers/custom-validators/activitypub/activity.js'
import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments.js'
import { PeerTubeRequestError } from '@server/helpers/requests.js'
import { AP_CLEANER } from '@server/initializers/constants.js'
import { fetchAP } from '@server/lib/activitypub/activity.js'
import { checkUrlsSameHost } from '@server/lib/activitypub/url.js'
import { Redis } from '@server/lib/redis.js'
import { VideoCommentModel } from '@server/models/video/video-comment.js'
import { VideoShareModel } from '@server/models/video/video-share.js'
import { VideoModel } from '@server/models/video/video.js'
import { HttpStatusCode } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js'
const lTags = loggerTagsFactory('ap-cleaner')
// Job to clean remote interactions off local videos
async function processActivityPubCleaner (_job: Job) {
logger.info('Processing ActivityPub cleaner.', lTags())
{
const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
const { bodyValidator, deleter, updater } = rateOptionsFactory()
await Bluebird.map(rateUrls, async rateUrl => {
// TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed
if (rateUrl.includes('#')) return
const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
if (result?.status === 'deleted') {
const { videoId, type } = result.data
await VideoModel.syncLocalRates(videoId, type, undefined)
}
}, { concurrency: AP_CLEANER.CONCURRENCY })
}
{
const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos()
const { bodyValidator, deleter, updater } = shareOptionsFactory()
await Bluebird.map(shareUrls, async shareUrl => {
await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
}, { concurrency: AP_CLEANER.CONCURRENCY })
}
{
const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos()
const { bodyValidator, deleter, updater } = commentOptionsFactory()
await Bluebird.map(commentUrls, async commentUrl => {
await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
}, { concurrency: AP_CLEANER.CONCURRENCY })
}
}
// ---------------------------------------------------------------------------
export {
processActivityPubCleaner
}
// ---------------------------------------------------------------------------
async function updateObjectIfNeeded <T> (options: {
url: string
bodyValidator: (body: any) => boolean
updater: (url: string, newUrl: string) => Promise<T>
deleter: (url: string) => Promise<T> }
): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
const { url, bodyValidator, updater, deleter } = options
const on404OrTombstone = async () => {
logger.info('Removing remote AP object %s.', url, lTags(url))
const data = await deleter(url)
return { status: 'deleted' as 'deleted', data }
}
try {
const { body } = await fetchAP<any>(url)
// If not same id, check same host and update
if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
if (body.type === 'Tombstone') {
return on404OrTombstone()
}
const newUrl = body.id
if (newUrl !== url) {
if (checkUrlsSameHost(newUrl, url) !== true) {
throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
}
logger.info('Updating remote AP object %s.', url, lTags(url))
const data = await updater(url, newUrl)
return { status: 'updated', data }
}
return null
} catch (err) {
// Does not exist anymore, remove entry
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
return on404OrTombstone()
}
logger.debug('Remote AP object %s is unavailable.', url, lTags(url))
const unavailability = await Redis.Instance.addAPUnavailability(url)
if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) {
logger.info('Removing unavailable AP resource %s.', url, lTags(url))
return on404OrTombstone()
}
return null
}
}
function rateOptionsFactory () {
return {
bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body),
updater: async (url: string, newUrl: string) => {
const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
rate.url = newUrl
const videoId = rate.videoId
const type = rate.type
await rate.save()
return { videoId, type }
},
deleter: async (url) => {
const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
const videoId = rate.videoId
const type = rate.type
await rate.destroy()
return { videoId, type }
}
}
}
function shareOptionsFactory () {
return {
bodyValidator: (body: any) => isAnnounceActivityValid(body),
updater: async (url: string, newUrl: string) => {
const share = await VideoShareModel.loadByUrl(url, undefined)
share.url = newUrl
await share.save()
return undefined
},
deleter: async (url) => {
const share = await VideoShareModel.loadByUrl(url, undefined)
await share.destroy()
return undefined
}
}
}
function commentOptionsFactory () {
return {
bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body),
updater: async (url: string, newUrl: string) => {
const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(url)
comment.url = newUrl
await comment.save()
return undefined
},
deleter: async (url) => {
const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(url)
await comment.destroy()
return undefined
}
}
}

View File

@@ -0,0 +1,103 @@
import { Job } from 'bullmq'
import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'
import { ActivitypubFollowPayload } from '@peertube/peertube-models'
import { sanitizeHost } from '../../../helpers/core-utils.js'
import { retryTransactionWrapper } from '../../../helpers/database-utils.js'
import { logger } from '../../../helpers/logger.js'
import { REMOTE_SCHEME, SERVER_ACTOR_NAME, WEBSERVER } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
import { ActorModel } from '../../../models/actor/actor.js'
import { ActorFollowModel } from '../../../models/actor/actor-follow.js'
import { MActor, MActorFull } from '../../../types/models/index.js'
import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors/index.js'
import { sendFollow } from '../../activitypub/send/index.js'
import { Notifier } from '../../notifier/index.js'
import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'
async function processActivityPubFollow (job: Job) {
const payload = job.data as ActivitypubFollowPayload
const host = payload.host
const identifier = [ payload.name || SERVER_ACTOR_NAME, host ]
.filter(v => !!v).join('@')
logger.info('Processing ActivityPub follow in job %s.', job.id)
let targetActor: MActorFull
if (!host || host === WEBSERVER.HOST) {
if (!payload.name) throw new Error('Payload name is mandatory for local follow')
targetActor = await ActorModel.loadLocalByName(payload.name)
} else {
const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP)
let actorUrl: string
try {
if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost)
if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost)
targetActor = await getOrCreateAPActor(actorUrl, 'all')
} catch (err) {
logger.warn(`Do not follow ${identifier} because we could not find the actor URL (in database or using webfinger)`, { err })
return
}
}
if (!targetActor) {
logger.warn(`Do not follow ${identifier} because we could not fetch/load the actor`)
return
}
if (payload.assertIsChannel && !targetActor.VideoChannel) {
logger.warn(`Do not follow ${identifier} because it is not a channel.`)
return
}
const fromActor = await ActorModel.load(payload.followerActorId)
return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow)
}
// ---------------------------------------------------------------------------
export {
processActivityPubFollow
}
// ---------------------------------------------------------------------------
async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) {
if (fromActor.id === targetActor.id) {
throw new Error('Follower is the same as target actor.')
}
// Same server, direct accept
const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending'
const actorFollow = await sequelizeTypescript.transaction(async t => {
const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({
byActor: fromActor,
state,
targetActor,
activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor),
transaction: t
})
// Send a notification to remote server if our follow is not already accepted
if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
return actorFollow
})
const followerFull = await ActorModel.loadFull(fromActor.id)
const actorFollowFull = Object.assign(actorFollow, {
ActorFollowing: targetActor,
ActorFollower: followerFull
})
if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull)
if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull)
return actorFollow
}

View File

@@ -0,0 +1,49 @@
import { Job } from 'bullmq'
import { ActivitypubHttpBroadcastPayload } from '@peertube/peertube-models'
import { buildGlobalHTTPHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/http.js'
import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache.js'
import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process.js'
import { logger } from '../../../helpers/logger.js'
// Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive
async function processActivityPubHttpSequentialBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) {
logger.info('Processing ActivityPub broadcast in job %s.', job.id)
const requestOptions = await buildRequestOptions(job.data)
const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
}
async function processActivityPubParallelHttpBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) {
logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id)
const requestOptions = await buildRequestOptions(job.data)
const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
}
// ---------------------------------------------------------------------------
export {
processActivityPubHttpSequentialBroadcast,
processActivityPubParallelHttpBroadcast
}
// ---------------------------------------------------------------------------
async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) {
const body = await computeBody(payload)
const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true })
return {
method: 'POST' as 'POST',
json: body,
httpSignature: httpSignatureOptions,
headers: await buildGlobalHTTPHeaders(body)
}
}

View File

@@ -0,0 +1,45 @@
import { Job } from 'bullmq'
import { ActivitypubHttpFetcherPayload, FetchType } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
import { VideoModel } from '../../../models/video/video.js'
import { VideoCommentModel } from '../../../models/video/video-comment.js'
import { VideoShareModel } from '../../../models/video/video-share.js'
import { MAccountDefault, MVideoFullLight } from '../../../types/models/index.js'
import { crawlCollectionPage } from '../../activitypub/crawl.js'
import { createAccountPlaylists } from '../../activitypub/playlists/index.js'
import { processActivities } from '../../activitypub/process/index.js'
import { addVideoShares } from '../../activitypub/share.js'
import { addVideoComments } from '../../activitypub/video-comments.js'
import { AccountModel } from '@server/models/account/account.js'
async function processActivityPubHttpFetcher (job: Job) {
logger.info('Processing ActivityPub fetcher in job %s.', job.id)
const payload = job.data as ActivitypubHttpFetcherPayload
let video: MVideoFullLight
if (payload.videoId) video = await VideoModel.loadFull(payload.videoId)
let account: MAccountDefault
if (payload.accountId) account = await AccountModel.load(payload.accountId)
const fetcherType: { [id in FetchType]: (items: any[]) => Promise<any> } = {
'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
'video-shares': items => addVideoShares(items, video),
'video-comments': items => addVideoComments(items),
'account-playlists': items => createAccountPlaylists(items, account)
}
const cleanerType: { [id in FetchType]?: (crawlStartDate: Date) => Promise<any> } = {
'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
}
return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
}
// ---------------------------------------------------------------------------
export {
processActivityPubHttpFetcher
}

View File

@@ -0,0 +1,38 @@
import { Job } from 'bullmq'
import { ActivitypubHttpUnicastPayload } from '@peertube/peertube-models'
import { buildGlobalHTTPHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/http.js'
import { logger } from '../../../helpers/logger.js'
import { ActorFollowHealthCache } from '../../actor-follow-health-cache.js'
import { httpUnicastFromWorker } from '@server/lib/worker/parent-process.js'
async function processActivityPubHttpUnicast (job: Job) {
logger.info('Processing ActivityPub unicast in job %s.', job.id)
const payload = job.data as ActivitypubHttpUnicastPayload
const uri = payload.uri
const body = await computeBody(payload)
const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true })
const options = {
method: 'POST' as 'POST',
json: body,
httpSignature: httpSignatureOptions,
headers: await buildGlobalHTTPHeaders(body)
}
try {
await httpUnicastFromWorker({ uri, requestOptions: options })
ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], [])
} catch (err) {
ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ])
throw err
}
}
// ---------------------------------------------------------------------------
export {
processActivityPubHttpUnicast
}

View File

@@ -0,0 +1,60 @@
import { RefreshPayload } from '@peertube/peertube-models'
import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists/index.js'
import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders/index.js'
import { Job } from 'bullmq'
import { logger } from '../../../helpers/logger.js'
import { ActorModel } from '../../../models/actor/actor.js'
import { VideoPlaylistModel } from '../../../models/video/video-playlist.js'
import { refreshActorIfNeeded } from '../../activitypub/actors/index.js'
async function refreshAPObject (job: Job) {
const payload = job.data as RefreshPayload
logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url)
if (payload.type === 'video') return refreshVideo(payload.url)
if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
if (payload.type === 'actor') return refreshActor(payload.url)
}
// ---------------------------------------------------------------------------
export {
refreshAPObject
}
// ---------------------------------------------------------------------------
async function refreshVideo (videoUrl: string) {
const fetchType = 'all'
const syncParam = { rates: true, shares: true, comments: true }
const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
if (videoFromDatabase) {
const refreshOptions = {
video: videoFromDatabase,
fetchedType: fetchType as VideoLoadByUrlType,
syncParam
}
await refreshVideoIfNeeded(refreshOptions)
}
}
async function refreshActor (actorUrl: string) {
const fetchType = 'all'
const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl)
if (actor) {
await refreshActorIfNeeded({ actor, fetchedType: fetchType })
}
}
async function refreshVideoPlaylist (playlistUrl: string) {
const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
if (playlist) {
await refreshVideoPlaylistIfNeeded(playlist)
}
}

View File

@@ -0,0 +1,20 @@
import { Job } from 'bullmq'
import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors/index.js'
import { ActorModel } from '@server/models/actor/actor.js'
import { ActorKeysPayload } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
async function processActorKeys (job: Job) {
const payload = job.data as ActorKeysPayload
logger.info('Processing actor keys in job %s.', job.id)
const actor = await ActorModel.load(payload.actorId)
await generateAndSaveActorKeys(actor)
}
// ---------------------------------------------------------------------------
export {
processActorKeys
}

View File

@@ -0,0 +1,39 @@
import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
import { Job } from 'bullmq'
const lTags = loggerTagsFactory('channel-synchronization')
export async function processAfterVideoChannelImport (job: Job) {
const payload = job.data as AfterVideoChannelImportPayload
if (!payload.channelSyncId) return
logger.info('Processing after video channel import in job %s.', job.id)
const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
if (!sync) {
logger.error('Unknown sync id %d.', payload.channelSyncId)
return
}
const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
let errors = payload.buildJobErrors || 0
let successes = 0
for (const value of Object.values(childrenValues)) {
if (value.resultType === 'success') successes++
else if (value.resultType === 'error') errors++
}
if (errors > 0) {
sync.state = VideoChannelSyncState.FAILED
logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes, ...lTags() })
} else {
sync.state = VideoChannelSyncState.SYNCED
logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes, ...lTags() })
}
await sync.save()
}

View File

@@ -0,0 +1,34 @@
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CreateUserExportPayload } from '@peertube/peertube-models'
import { UserExportModel } from '@server/models/user/user-export.js'
import { UserExporter } from '@server/lib/user-import-export/user-exporter.js'
import { Emailer } from '@server/lib/emailer.js'
const lTags = loggerTagsFactory('user-export')
export async function processCreateUserExport (job: Job): Promise<void> {
const payload = job.data as CreateUserExportPayload
const exportModel = await UserExportModel.load(payload.userExportId)
logger.info('Processing create user export %s in job %s.', payload.userExportId, job.id, lTags())
if (!exportModel) {
logger.info(`User export ${payload.userExportId} does not exist anymore, do not create user export.`, lTags())
return
}
const exporter = new UserExporter()
try {
await exporter.export(exportModel)
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
logger.info(`User export ${payload.userExportId} has been created`, lTags())
} catch (err) {
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
throw err
}
}

View File

@@ -0,0 +1,11 @@
import { Job } from 'bullmq'
import { EmailPayload } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
import { Emailer } from '../../emailer.js'
export function processEmail (job: Job) {
const payload = job.data as EmailPayload
logger.info('Processing email in job %s.', job.id)
return Emailer.Instance.sendMail(payload)
}

View File

@@ -0,0 +1,28 @@
import { Job } from 'bullmq'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { VideoModel } from '@server/models/video/video.js'
import { FederateVideoPayload } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
function processFederateVideo (job: Job) {
const payload = job.data as FederateVideoPayload
logger.info('Processing video federation in job %s.', job.id)
return retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async t => {
const video = await VideoModel.loadFull(payload.videoUUID, t)
if (!video) return
return federateVideoIfNeeded(video, payload.isNewVideoForFederation, t)
})
})
}
// ---------------------------------------------------------------------------
export {
processFederateVideo
}

View File

@@ -0,0 +1,100 @@
import { FFmpegImage } from '@peertube/peertube-ffmpeg'
import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
import { generateImageFilename } from '@server/helpers/image-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { STORYBOARD } from '@server/initializers/constants.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
import { VideoModel } from '@server/models/video/video.js'
import { Job } from 'bullmq'
import { join } from 'path'
import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
const lTagsBase = loggerTagsFactory('storyboard')
export async function processGenerateStoryboard (job: Job): Promise<void> {
const payload = job.data as GenerateStoryboardPayload
const lTags = lTagsBase(payload.videoUUID)
logger.info(`Processing generate storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
if (CONFIG.STORYBOARDS.ENABLED !== true) {
logger.info(`Storyboard disabled, do not process storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
return
}
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
try {
const video = await VideoModel.loadFull(payload.videoUUID)
if (!video) {
logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
return
}
const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
if (!inputFile) {
logger.info(`Do not generate a storyboard of ${payload.videoUUID} since the video does not have a video stream`, lTags)
return
}
await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
const { totalSprites, spriteDuration } = buildTotalSprites(video)
if (totalSprites === 0) {
logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)
return
}
const spritesCount = findGridSize({
toFind: totalSprites,
maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
})
const filename = generateImageFilename()
const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
logger.debug(
`Generating storyboard from video of ${video.uuid} to ${destination}`,
{ ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
)
const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
await ffmpeg.generateStoryboardFromVideo({
destination,
path: videoPath,
inputFileMutexReleaser,
sprites: {
size: {
height: spriteHeight,
width: spriteWidth
},
count: spritesCount,
duration: spriteDuration
}
})
await insertStoryboardInDatabase({
videoUUID: video.uuid,
lTags,
filename,
destination,
imageSize: await getImageSizeFromWorker(destination),
spriteHeight,
spriteWidth,
spriteDuration,
federate: payload.federate
})
})
} finally {
inputFileMutexReleaser()
}
}

View File

@@ -0,0 +1,33 @@
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { ImportUserArchivePayload } from '@peertube/peertube-models'
import { UserImportModel } from '@server/models/user/user-import.js'
import { UserImporter } from '@server/lib/user-import-export/user-importer.js'
import { Emailer } from '@server/lib/emailer.js'
const lTags = loggerTagsFactory('user-import')
export async function processImportUserArchive (job: Job): Promise<void> {
const payload = job.data as ImportUserArchivePayload
const importModel = await UserImportModel.load(payload.userImportId)
logger.info(`Processing importing user archive ${payload.userImportId} in job ${job.id}`, lTags())
if (!importModel) {
logger.info(`User import ${payload.userImportId} does not exist anymore, do not create import data.`, lTags())
return
}
const exporter = new UserImporter()
await exporter.import(importModel)
try {
await Emailer.Instance.addUserImportSuccessJob(importModel)
logger.info(`User import ${payload.userImportId} ended`, lTags())
} catch (err) {
await Emailer.Instance.addUserImportErroredJob(importModel)
throw err
}
}

View File

@@ -0,0 +1,64 @@
import { LiveVideoStartPayload } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { startScheduledLiveSession } from '@server/controllers/api/live-stream.js'
import { CONFIG } from '@server/initializers/config.js'
import { LiveVideoProcessManager } from '@server/lib/live-videos/live-video-process-manager.js'
import { LIVE_VIDEO_START_LEAD_TIME_MS } from '@server/lib/live-videos/shared.js'
import { LiveVideoModel, LiveVideoStatus } from '@server/models/video/live-video.js'
import { Job } from 'bullmq'
import { setTimeout as wait } from 'node:timers/promises'
const lTags = loggerTagsFactory('jobs', 'live-video-start')
const LIVE_VIDEO_START_POLL_MS = 5_000
export async function processLiveVideoStart (job: Job) {
const payload = job.data as LiveVideoStartPayload
let liveVideo: LiveVideoModel
while (true) {
const currentLiveVideo = await LiveVideoModel.loadById(payload.liveVideoId)
if (!currentLiveVideo) {
logger.warn('Cannot find live video %d for scheduled start job %s.', payload.liveVideoId, job.id, lTags())
return
}
if (currentLiveVideo.status !== LiveVideoStatus.SCHEDULED) {
logger.info('Skipping scheduled start of live video %d because status is %s.', currentLiveVideo.id, currentLiveVideo.status, lTags())
return
}
const now = Date.now()
const startAt = currentLiveVideo.startTime.getTime()
const endAt = currentLiveVideo.endTime.getTime()
if (endAt <= now) {
logger.info('Skipping scheduled start of live video %d because it is already finished.', currentLiveVideo.id, lTags())
return
}
const launchAt = startAt - LIVE_VIDEO_START_LEAD_TIME_MS
const waitMs = launchAt - now
if (waitMs <= 0) {
liveVideo = currentLiveVideo
break
}
await wait(Math.min(waitMs, LIVE_VIDEO_START_POLL_MS))
}
if (CONFIG.LIVE.ENABLE_SOURCE_START) {
await LiveVideoProcessManager.Instance.start(liveVideo)
} else {
logger.info('Skipping source-side scheduled live start for live video %d because live.enable_source_start is disabled.', liveVideo.id, lTags())
}
const sinkWaitMs = liveVideo.startTime.getTime() - Date.now()
if (sinkWaitMs > 0) {
await wait(sinkWaitMs)
}
if (CONFIG.LIVE.ENABLE_SINK_START) {
await startScheduledLiveSession(liveVideo.targetCamera, { keepAliveUntil: liveVideo.endTime })
} else {
logger.info('Skipping sink-side scheduled live start for live video %d because live.enable_sink_start is disabled.', liveVideo.id, lTags())
}
}

View File

@@ -0,0 +1,110 @@
import { Job } from 'bullmq'
import { extractVideo } from '@server/helpers/video.js'
import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/lib/webtorrent.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoModel } from '@server/models/video/video.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import { ManageVideoTorrentPayload } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
async function processManageVideoTorrent (job: Job) {
const payload = job.data as ManageVideoTorrentPayload
logger.info('Processing torrent in job %s.', job.id)
if (payload.action === 'create') return doCreateAction(payload)
if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload)
}
// ---------------------------------------------------------------------------
export {
processManageVideoTorrent
}
// ---------------------------------------------------------------------------
async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) {
const [ video, file ] = await Promise.all([
loadVideoOrLog(payload.videoId),
loadFileOrLog(payload.videoFileId)
])
if (!video || !file) return
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
await video.reload()
await file.reload()
await createTorrentAndSetInfoHash(video, file)
// Refresh videoFile because the createTorrentAndSetInfoHash could be long
const refreshedFile = await VideoFileModel.loadWithVideo(file.id)
// File does not exist anymore, remove the generated torrent
if (!refreshedFile) return file.removeTorrent()
refreshedFile.infoHash = file.infoHash
refreshedFile.torrentFilename = file.torrentFilename
await refreshedFile.save()
} finally {
fileMutexReleaser()
}
}
async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) {
const [ video, streamingPlaylist, file ] = await Promise.all([
loadVideoOrLog(payload.videoId),
loadStreamingPlaylistOrLog(payload.streamingPlaylistId),
loadFileOrLog(payload.videoFileId)
])
if ((!video && !streamingPlaylist) || !file) return
const extractedVideo = extractVideo(video || streamingPlaylist)
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid)
try {
await updateTorrentMetadata(video || streamingPlaylist, file)
await file.save()
} finally {
fileMutexReleaser()
}
}
async function loadVideoOrLog (videoId: number) {
if (!videoId) return undefined
const video = await VideoModel.load(videoId)
if (!video) {
logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId)
}
return video
}
async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
if (!streamingPlaylistId) return undefined
const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId)
if (!streamingPlaylist) {
logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId)
}
return streamingPlaylist
}
async function loadFileOrLog (videoFileId: number) {
if (!videoFileId) return undefined
const file = await VideoFileModel.load(videoFileId)
if (!file) {
logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)
}
return file
}

View File

@@ -0,0 +1,45 @@
import { isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { moveCaptionToFS, moveVideoToFS, onMoveVideoToFSFailure } from '@server/lib/move-storage/move-to-file-system.js'
import { Job } from 'bullmq'
const lTagsBase = loggerTagsFactory('move-file-system')
export async function processMoveToFileSystem (job: Job) {
const payload = job.data as MoveStoragePayload
if (isMoveVideoStoragePayload(payload)) { // Move all video related files
logger.info(`Moving video ${payload.videoUUID} to file system in job ${job.id}.`, lTagsBase(payload.videoUUID))
const moveVideoState = payload.isNewVideo !== undefined
? { isNewVideo: payload.isNewVideo, previousVideoState: payload.previousVideoState }
: payload.moveVideoState
return moveVideoToFS({
videoUUID: payload.videoUUID,
moveVideoState,
loggerTags: lTagsBase().tags
})
} else if (isMoveCaptionPayload(payload)) { // Only caption file
logger.info(`Moving video caption ${payload.captionId} to file system in job ${job.id}.`, lTagsBase(payload.captionId))
return moveCaptionToFS({
captionId: payload.captionId,
loggerTags: lTagsBase().tags
})
} else {
throw new Error('Unknown payload type')
}
}
export async function onMoveToFileSystemFailure (job: Job, err: any) {
const payload = job.data as MoveStoragePayload
if (!isMoveVideoStoragePayload(payload)) return
await onMoveVideoToFSFailure({
videoUUID: payload.videoUUID,
err,
loggerTags: lTagsBase().tags
})
}

View File

@@ -0,0 +1,49 @@
import { isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload } from '@peertube/peertube-models'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import {
moveCaptionToObjectStorage,
moveVideoToObjectStorage,
onMoveVideoToObjectStorageFailure
} from '@server/lib/move-storage/move-to-object-storage.js'
import { Job } from 'bullmq'
const lTagsBase = loggerTagsFactory('move-object-storage')
export async function processMoveToObjectStorage (job: Job) {
const payload = job.data as MoveStoragePayload
if (isMoveVideoStoragePayload(payload)) { // Move all video related files
logger.info(`Moving video ${payload.videoUUID} to object storage in job ${job.id}`, lTagsBase(payload.videoUUID))
const moveVideoState = payload.isNewVideo !== undefined
? { isNewVideo: payload.isNewVideo, previousVideoState: payload.previousVideoState }
: payload.moveVideoState
await moveVideoToObjectStorage({
videoUUID: payload.videoUUID,
moveVideoState,
loggerTags: lTagsBase().tags
})
} else if (isMoveCaptionPayload(payload)) { // Only caption file
logger.info(`Moving video caption ${payload.captionId} to object storage in job ${job.id}.`, lTagsBase(payload.captionId))
return moveCaptionToObjectStorage({
captionId: payload.captionId,
loggerTags: lTagsBase().tags
})
} else {
throw new Error('Unknown payload type')
}
}
export async function onMoveToObjectStorageFailure (job: Job, err: any) {
const payload = job.data as MoveStoragePayload
if (!isMoveVideoStoragePayload(payload)) return
await onMoveVideoToObjectStorageFailure({
videoUUID: payload.videoUUID,
err,
loggerTags: lTagsBase().tags
})
}

View File

@@ -0,0 +1,27 @@
import { Job } from 'bullmq'
import { Notifier } from '@server/lib/notifier/index.js'
import { VideoModel } from '@server/models/video/video.js'
import { NotifyPayload } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
async function processNotify (job: Job) {
const payload = job.data as NotifyPayload
logger.info('Processing %s notification in job %s.', payload.action, job.id)
if (payload.action === 'new-video') return doNotifyNewVideo(payload)
}
// ---------------------------------------------------------------------------
export {
processNotify
}
// ---------------------------------------------------------------------------
async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) {
const refreshedVideo = await VideoModel.loadFull(payload.videoUUID)
if (!refreshedVideo) return
Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(refreshedVideo)
}

View File

@@ -0,0 +1,53 @@
import { pick } from '@peertube/peertube-core-utils'
import { TranscodingJobBuilderPayload, VideoFileStream } from '@peertube/peertube-models'
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { VideoModel } from '@server/models/video/video.js'
import { Job } from 'bullmq'
import { logger } from '../../../helpers/logger.js'
import { JobQueue } from '../job-queue.js'
/**
* Create transcoding jobs.
* They are not created directly so we can keep parent/children relationships using bullmq features
* while having the ability to use remote runner jobs that are not part of bullmq
*/
async function processTranscodingJobBuilder (job: Job) {
const payload = job.data as TranscodingJobBuilderPayload
logger.info('Processing transcoding job builder in job %s.', job.id)
if (payload.optimizeJob) {
const video = await VideoModel.loadFull(payload.videoUUID)
const user = await UserModel.loadByVideoId(video.id)
const videoFile = video.getMaxQualityFile(VideoFileStream.VIDEO) || video.getMaxQualityFile(VideoFileStream.AUDIO)
await createOptimizeOrMergeAudioJobs({
...pick(payload.optimizeJob, [ 'isNewVideo' ]),
video,
videoFile,
user
})
}
for (const job of (payload.jobs || [])) {
await JobQueue.Instance.createJob(job)
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
}
for (const sequentialJobs of (payload.sequentialJobs || [])) {
await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs)
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length)
}
}
// ---------------------------------------------------------------------------
export {
processTranscodingJobBuilder
}

View File

@@ -0,0 +1,43 @@
import { Job } from 'bullmq'
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { synchronizeChannel } from '@server/lib/sync-channel.js'
import { VideoChannelModel } from '@server/models/video/video-channel.js'
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
import { MChannelSync } from '@server/types/models/index.js'
import { VideoChannelImportPayload } from '@peertube/peertube-models'
export async function processVideoChannelImport (job: Job) {
const payload = job.data as VideoChannelImportPayload
logger.info('Processing video channel import in job %s.', job.id)
// Channel import requires only http upload to be allowed
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
throw new Error('Cannot import channel as the HTTP upload is disabled')
}
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
throw new Error('Cannot import channel as the synchronization is disabled')
}
let channelSync: MChannelSync
if (payload.partOfChannelSyncId) {
channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId)
if (!channelSync) {
throw new Error('Unknown channel sync specified in videos channel import')
}
}
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
await synchronizeChannel({
channel: videoChannel,
externalChannelUrl: payload.externalChannelUrl,
channelSync,
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
})
}

View File

@@ -0,0 +1,78 @@
import { Job } from 'bullmq'
import { copy } from 'fs-extra/esm'
import { VideoFileImportPayload } from '@peertube/peertube-models'
import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
import { CONFIG } from '@server/initializers/config.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideoFullLight } from '@server/types/models/index.js'
import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
import { logger } from '../../../helpers/logger.js'
import { JobQueue } from '../job-queue.js'
import { buildMoveVideoJob } from '@server/lib/video-jobs.js'
import { buildNewFile } from '@server/lib/video-file.js'
async function processVideoFileImport (job: Job) {
const payload = job.data as VideoFileImportPayload
logger.info('Processing video file import in job %s.', job.id)
const video = await VideoModel.loadFull(payload.videoUUID)
// No video, maybe deleted?
if (!video) {
logger.info(`Do not process job ${job.id}, video does not exist.`)
return undefined
}
await updateVideoFile(video, payload.filePath)
if (CONFIG.OBJECT_STORAGE.ENABLED) {
await JobQueue.Instance.createJob(
await buildMoveVideoJob({
type: 'move-to-object-storage',
video,
moveVideoState: {
isNewVideo: false,
previousVideoState: video.state
}
})
)
} else {
await federateVideoIfNeeded(video, false)
}
return video
}
// ---------------------------------------------------------------------------
export {
processVideoFileImport
}
// ---------------------------------------------------------------------------
async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath)
const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution)
if (currentVideoFile) {
// Remove old file and old torrent
await video.removeWebVideoFile(currentVideoFile)
// Remove the old video file from the array
video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
await currentVideoFile.destroy()
}
const newVideoFile = await buildNewFile({ mode: 'web-video', path: inputFilePath })
newVideoFile.videoId = video.id
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
await copy(inputFilePath, outputPath)
video.VideoFiles.push(newVideoFile)
await createTorrentAndSetInfoHash(video, newVideoFile)
await newVideoFile.save()
}

View File

@@ -0,0 +1,372 @@
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { ffprobePromise, getChaptersFromContainer, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import {
ThumbnailType,
ThumbnailType_Type,
VideoImportPayload,
VideoImportPreventExceptionResult,
VideoImportState,
VideoImportTorrentPayload,
VideoImportTorrentPayloadType,
VideoImportYoutubeDLPayload,
VideoImportYoutubeDLPayloadType,
VideoState
} from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
import { CONFIG } from '@server/initializers/config.js'
import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
import { setAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatic-tags.js'
import { isPostImportVideoAccepted } from '@server/lib/moderation.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { ServerConfigManager } from '@server/lib/server-config-manager.js'
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
import { Job } from 'bullmq'
import { FfprobeData } from 'fluent-ffmpeg'
import { move, remove } from 'fs-extra/esm'
import { stat } from 'fs/promises'
import { logger } from '../../../helpers/logger.js'
import { getSecureTorrentName } from '../../../helpers/utils.js'
import { CONSTRAINTS_FIELDS, JOB_TTL } from '../../../initializers/constants.js'
import { sequelizeTypescript } from '../../../initializers/database.js'
import { VideoFileModel } from '../../../models/video/video-file.js'
import { VideoImportModel } from '../../../models/video/video-import.js'
import { VideoModel } from '../../../models/video/video.js'
import { federateVideoIfNeeded } from '../../activitypub/videos/index.js'
import { Notifier } from '../../notifier/index.js'
import { generateLocalVideoMiniature } from '../../thumbnail.js'
import { JobQueue } from '../job-queue.js'
async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
const payload = job.data as VideoImportPayload
const videoImport = await getVideoImportOrDie(payload)
if (videoImport.state === VideoImportState.CANCELLED) {
logger.info('Do not process import since it has been cancelled', { payload })
return { resultType: 'success' }
}
if (videoImport.attempts >= CONFIG.IMPORT.VIDEOS.MAX_ATTEMPTS) {
logger.info('Do not process import since it has reached the maximum number of attempts', { payload, attempts: videoImport.attempts })
return { resultType: 'error' }
}
videoImport.attempts += 1
videoImport.state = VideoImportState.PROCESSING
await videoImport.save()
try {
if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
return { resultType: 'success' }
} catch (err) {
// Processors already handle video import state change on error
if (!payload.preventException) throw err
logger.warn('Catch error in video import to send value to parent job.', { payload, err })
return { resultType: 'error' }
}
}
// ---------------------------------------------------------------------------
export {
processVideoImport
}
// ---------------------------------------------------------------------------
async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) {
logger.info('Processing torrent video import in job %s.', job.id)
const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }
const target = {
torrentName: videoImport.torrentName
? getSecureTorrentName(videoImport.torrentName)
: undefined,
uri: videoImport.magnetUri
}
return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)
}
async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {
logger.info('Processing youtubeDL video import in job %s.', job.id)
const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id }
const youtubeDL = new YoutubeDLWrapper(
videoImport.targetUrl,
ServerConfigManager.Instance.getEnabledResolutions('vod'),
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
)
return processFile(
() => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),
videoImport,
options
)
}
async function getVideoImportOrDie (payload: VideoImportPayload) {
const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
if (!videoImport) throw new Error('Video import not found')
if (!videoImport.Video) {
const err = new Error(
`Cannot process video import ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`
)
await onImportError(err, null, videoImport)
throw err
}
return videoImport
}
type ProcessFileOptions = {
type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType
generateTranscription: boolean
videoImportId: number
}
async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
let tmpVideoPath: string
let videoFile: MVideoFile
try {
// Download video from youtubeDL
tmpVideoPath = await downloader()
// Get information about this video
const stats = await stat(tmpVideoPath)
const isAble = await isUserQuotaValid({ userId: videoImport.User.id, uploadSize: stats.size })
if (isAble === false) {
throw new Error('The user video quota is exceeded with this video to import.')
}
const ffprobe = await ffprobePromise(tmpVideoPath)
const duration = await getVideoStreamDuration(tmpVideoPath, ffprobe)
const containerChapters = await getChaptersFromContainer({
path: tmpVideoPath,
maxTitleLength: CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE.max,
ffprobe
})
videoFile = await buildNewFile({ mode: 'web-video', ffprobe, path: tmpVideoPath })
videoFile.videoId = videoImport.videoId
const hookName = options.type === 'youtube-dl'
? 'filter:api.video.post-import-url.accept.result'
: 'filter:api.video.post-import-torrent.accept.result'
// Check we accept this video
const acceptParameters = {
videoImport,
video: videoImport.Video,
videoFilePath: tmpVideoPath,
videoFile: videoFile as VideoFileModel,
user: videoImport.User
}
const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)
if (acceptedResult.accepted !== true) {
logger.info('Refused imported video.', { acceptedResult, acceptParameters })
videoImport.state = VideoImportState.REJECTED
await videoImport.save()
throw new Error(acceptedResult.errorMessage)
}
// Video is accepted, resuming preparation
const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid)
try {
const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile)
// Move file
const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
await move(tmpVideoPath, videoDestFile)
tmpVideoPath = null // This path is not used anymore
const thumbnails = await generateMiniature({ videoImportWithFiles, videoFile, ffprobe })
// Create torrent
await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async t => {
// Refresh video
const video = await VideoModel.load(videoImportWithFiles.videoId, t)
if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
await videoFile.save({ transaction: t })
// Update video DB object
video.duration = duration
video.state = buildNextVideoState(video.state)
video.aspectRatio = buildAspectRatio({ width: videoFile.width, height: videoFile.height })
await video.save({ transaction: t })
for (const thumbnail of thumbnails) {
await video.addAndSaveThumbnail(thumbnail, t)
}
await replaceChaptersIfNotExist({ video, chapters: containerChapters, transaction: t })
const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video, transaction: t })
await setAndSaveVideoAutomaticTags({ video, automaticTags, transaction: t })
// Now we can federate the video (reload from database, we need more attributes)
const videoForFederation = await VideoModel.loadFull(video.uuid, t)
await federateVideoIfNeeded(videoForFederation, true, t)
// Update video import object
videoImportWithFiles.state = VideoImportState.SUCCESS
const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport
logger.info('Video %s imported.', video.uuid)
return { videoImportUpdated, video: videoForFederation }
})
})
await afterImportSuccess({
videoImport: videoImportUpdated,
video,
videoFile,
user: videoImport.User,
generateTranscription: options.generateTranscription
})
} finally {
videoFileLockReleaser()
}
} catch (err) {
await onImportError(err, tmpVideoPath, videoImport)
throw err
}
}
async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> {
// Refresh video, privacy may have changed
const video = await videoImport.Video.reload()
const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
return Object.assign(videoImport, { Video: videoWithFiles })
}
async function generateMiniature (options: {
videoImportWithFiles: MVideoImportDefaultFiles
videoFile: MVideoFile
ffprobe: FfprobeData
}) {
const { ffprobe, videoFile, videoImportWithFiles } = options
const thumbnailsToGenerate: ThumbnailType_Type[] = []
if (!videoImportWithFiles.Video.getMiniature()) {
thumbnailsToGenerate.push(ThumbnailType.MINIATURE)
}
if (!videoImportWithFiles.Video.getPreview()) {
thumbnailsToGenerate.push(ThumbnailType.PREVIEW)
}
return generateLocalVideoMiniature({
video: videoImportWithFiles.Video,
videoFile,
types: thumbnailsToGenerate,
ffprobe
})
}
async function afterImportSuccess (options: {
videoImport: MVideoImport
video: MVideoFullLight
videoFile: MVideoFile
user: MUserId
generateTranscription: boolean
}) {
const { video, videoFile, videoImport, user, generateTranscription } = options
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
if (video.isBlacklisted()) {
const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
} else {
Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)
}
// Generate the storyboard in the job queue, and don't forget to federate an update after
await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
if (await VideoCaptionModel.hasVideoCaption(video.id) !== true && generateTranscription === true) {
await createTranscriptionTaskIfNeeded(video)
}
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
await JobQueue.Instance.createJob(
await buildMoveVideoJob({
type: 'move-to-object-storage',
video,
moveVideoState: {
isNewVideo: true,
previousVideoState: VideoState.TO_IMPORT
}
})
)
return
}
if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user })
}
}
async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) {
try {
if (tempVideoPath) await remove(tempVideoPath)
} catch (errUnlink) {
logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
}
await sequelizeTypescript.transaction(async t => {
videoImport.error = err.message
if (videoImport.state !== VideoImportState.REJECTED) {
videoImport.state = VideoImportState.FAILED
}
await videoImport.save({ transaction: t })
const video = await VideoModel.load(videoImport.videoId, t)
if (video) {
video.state = VideoState.TO_IMPORT_FAILED
await video.save({ transaction: t })
}
})
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
}

View File

@@ -0,0 +1,369 @@
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
import { ThumbnailType, VideoFileStream, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models'
import { peertubeTruncate } from '@server/helpers/core-utils.js'
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live/index.js'
import {
generateHLSMasterPlaylistFilename,
generateHlsSha256SegmentsFilename,
getHLSDirectory,
getLiveReplayBaseDirectory
} from '@server/lib/paths.js'
import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
import { moveToNextState } from '@server/lib/video-state.js'
import { setVideoTags } from '@server/lib/video.js'
import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js'
import { VideoLiveModel } from '@server/models/video/video-live.js'
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
import { VideoModel } from '@server/models/video/video.js'
import {
MThumbnail,
MVideo,
MVideoLive,
MVideoLiveSession,
MVideoTag,
MVideoThumbnail,
MVideoWithAllFiles,
MVideoWithFileThumbnail
} from '@server/types/models/index.js'
import { Job } from 'bullmq'
import { pathExists, remove } from 'fs-extra/esm'
import { readdir } from 'fs/promises'
import { isAbsolute, join } from 'path'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
const lTags = loggerTagsFactory('live', 'job')
export async function processVideoLiveEnding (job: Job) {
const payload = job.data as VideoLiveEndingPayload
logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() })
function logError () {
logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags())
}
const video = await VideoModel.load(payload.videoId)
const live = await VideoLiveModel.loadByVideoId(payload.videoId)
const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId)
if (!video || !live || !liveSession) {
logError()
return
}
const permanentLive = live.permanentLive
liveSession.endingProcessed = true
await liveSession.save()
if (liveSession.saveReplay !== true) {
return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
}
let replayDirectory = payload.replayDirectory
// Introduced in PeerTube 7.2, allow to use the appropriate base directory even if the live privacy changed
if (!isAbsolute(replayDirectory)) {
replayDirectory = join(getLiveReplayBaseDirectory(video), replayDirectory)
}
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
try {
await video.reload()
if (await hasReplayFiles(replayDirectory) !== true) {
logger.info(`No replay files found for live ${video.uuid}, skipping video replay creation.`, { ...lTags(video.uuid) })
await cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
} else if (permanentLive) {
await saveReplayToExternalVideo({
liveVideo: video,
liveSession,
publishedAt: payload.publishedAt,
replayDirectory
})
await cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
} else {
await replaceLiveByReplay({
video,
liveSession,
live,
permanentLive,
replayDirectory
})
}
} finally {
inputFileMutexReleaser()
}
}
// ---------------------------------------------------------------------------
// Private
// ---------------------------------------------------------------------------
async function saveReplayToExternalVideo (options: {
liveVideo: MVideoThumbnail
liveSession: MVideoLiveSession
publishedAt: string
replayDirectory: string
}) {
const { liveSession, publishedAt, replayDirectory } = options
const liveVideo = await VideoModel.loadFull(options.liveVideo.id)
const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId)
const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}`
const truncatedVideoName = peertubeTruncate(liveVideo.name, {
length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length
})
const replayVideo = new VideoModel({
name: truncatedVideoName + videoNameSuffix,
isLive: false,
state: VideoState.TO_TRANSCODE,
duration: 0,
remote: liveVideo.remote,
category: liveVideo.category,
licence: liveVideo.licence,
language: liveVideo.language,
commentsPolicy: liveVideo.commentsPolicy,
downloadEnabled: liveVideo.downloadEnabled,
waitTranscoding: true,
nsfw: liveVideo.nsfw,
description: liveVideo.description,
aspectRatio: liveVideo.aspectRatio,
support: liveVideo.support,
privacy: replaySettings.privacy,
channelId: liveVideo.channelId
}) as MVideoWithAllFiles & MVideoTag
replayVideo.Thumbnails = []
replayVideo.VideoFiles = []
replayVideo.VideoStreamingPlaylists = []
replayVideo.url = getLocalVideoActivityPubUrl(replayVideo)
await replayVideo.save()
await setVideoTags({ video: replayVideo, tags: liveVideo.Tags.map(t => t.name) })
liveSession.replayVideoId = replayVideo.id
await liveSession.save()
// If live is blacklisted, also blacklist the replay
const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id)
if (blacklist) {
await VideoBlacklistModel.create({
videoId: replayVideo.id,
unfederated: blacklist.unfederated,
reason: blacklist.reason,
type: blacklist.type
})
}
await assignReplayFilesToVideo({ video: replayVideo, replayDirectory })
logger.info(`Removing replay directory ${replayDirectory}`, lTags(liveVideo.uuid))
await remove(replayDirectory)
try {
await copyOrRegenerateThumbnails({ liveVideo, replayVideo })
} catch (err) {
logger.error(
`Cannot copy/regenerate thumbnails of ended live ${liveVideo.uuid} to external video ${replayVideo.uuid}`,
lTags(liveVideo.uuid, replayVideo.uuid)
)
}
await createStoryboardJob(replayVideo)
await createTranscriptionTaskIfNeeded(replayVideo)
await moveToNextState({ video: replayVideo, isNewVideo: true })
}
async function copyOrRegenerateThumbnails (options: {
liveVideo: MVideoThumbnail
replayVideo: MVideoWithFileThumbnail
}) {
const { liveVideo, replayVideo } = options
let thumbnails: MThumbnail[] = []
const preview = liveVideo.getPreview()
if (preview?.automaticallyGenerated === false) {
thumbnails = await Promise.all(
[ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ].map(type => {
return updateLocalVideoMiniatureFromExisting({
inputPath: preview.getPath(),
video: replayVideo,
type,
automaticallyGenerated: false,
keepOriginal: true
})
})
)
} else {
thumbnails = await generateLocalVideoMiniature({
video: replayVideo,
videoFile: replayVideo.getMaxQualityFile(VideoFileStream.VIDEO) || replayVideo.getMaxQualityFile(VideoFileStream.AUDIO),
types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ],
ffprobe: undefined
})
}
for (const thumbnail of thumbnails) {
await replayVideo.addAndSaveThumbnail(thumbnail)
}
}
async function replaceLiveByReplay (options: {
video: MVideo
liveSession: MVideoLiveSession
live: MVideoLive
permanentLive: boolean
replayDirectory: string
}) {
const { video: liveVideo, liveSession, live, permanentLive, replayDirectory } = options
const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId)
const videoWithFiles = await VideoModel.loadFull(liveVideo.id)
const hlsPlaylist = videoWithFiles.getHLSPlaylist()
const replayInAnotherDirectory = isVideoInPublicDirectory(liveVideo.privacy) !== isVideoInPublicDirectory(replaySettings.privacy)
logger.info(`Replacing live ${liveVideo.uuid} by replay ${replayDirectory}.`, { replayInAnotherDirectory, ...lTags(liveVideo.uuid) })
await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist)
await live.destroy()
videoWithFiles.isLive = false
videoWithFiles.privacy = replaySettings.privacy
videoWithFiles.waitTranscoding = true
videoWithFiles.state = VideoState.TO_TRANSCODE
await videoWithFiles.save()
liveSession.replayVideoId = videoWithFiles.id
await liveSession.save()
await VideoFileModel.removeHLSFilesOfStreamingPlaylistId(hlsPlaylist.id)
// Reset playlist
hlsPlaylist.VideoFiles = []
hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
await hlsPlaylist.save()
await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
// Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay
if (permanentLive) { // Remove session replay
await remove(replayDirectory)
} else {
// We won't stream again in this live, we can delete the base replay directory
await remove(getLiveReplayBaseDirectory(liveVideo))
// If the live was in another base directory, also delete it
if (replayInAnotherDirectory) {
await remove(getHLSDirectory(liveVideo))
}
}
// Regenerate the thumbnail & preview?
try {
await regenerateMiniaturesIfNeeded(videoWithFiles, undefined)
} catch (err) {
logger.error(`Cannot regenerate thumbnails of ended live ${videoWithFiles.uuid}`, lTags(liveVideo.uuid))
}
// We consider this is a new video
await moveToNextState({ video: videoWithFiles, isNewVideo: true })
await createStoryboardJob(videoWithFiles)
await createTranscriptionTaskIfNeeded(videoWithFiles)
}
async function assignReplayFilesToVideo (options: {
video: MVideo
replayDirectory: string
}) {
const { video, replayDirectory } = options
const concatenatedTsFiles = await readdir(replayDirectory)
logger.info(`Assigning replays ${replayDirectory} to video ${video.uuid}.`, { concatenatedTsFiles, ...lTags(video.uuid) })
for (const concatenatedTsFile of concatenatedTsFiles) {
// Generating hls playlist can be long, reload the video in this case
await video.reload()
const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
const probe = await ffprobePromise(concatenatedTsFilePath)
const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe)
try {
await generateHlsPlaylistResolutionFromTS({
video,
inputFileMutexReleaser: null, // Already locked in parent
concatenatedTsFilePath,
resolution,
fps,
isAAC: audioStream?.codec_name === 'aac'
})
} catch (err) {
logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
}
}
return video
}
async function cleanupLiveAndFederate (options: {
video: MVideo
permanentLive: boolean
streamingPlaylistId: number
}) {
const { permanentLive, video, streamingPlaylistId } = options
const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId)
if (streamingPlaylist) {
if (permanentLive) {
await cleanupAndDestroyPermanentLive(video, streamingPlaylist)
} else {
await cleanupUnsavedNormalLive(video, streamingPlaylist)
}
}
try {
const fullVideo = await VideoModel.loadFull(video.id)
return federateVideoIfNeeded(fullVideo, false, undefined)
} catch (err) {
logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
}
}
function createStoryboardJob (video: MVideo) {
return addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
}
async function hasReplayFiles (replayDirectory: string) {
return await pathExists(replayDirectory) && (await readdir(replayDirectory)).length !== 0
}

View File

@@ -0,0 +1,17 @@
import { Job } from 'bullmq'
import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler.js'
import { VideoRedundancyPayload } from '@peertube/peertube-models'
import { logger } from '../../../helpers/logger.js'
async function processVideoRedundancy (job: Job) {
const payload = job.data as VideoRedundancyPayload
logger.info('Processing video redundancy in job %s.', job.id)
return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
}
// ---------------------------------------------------------------------------
export {
processVideoRedundancy
}

View File

@@ -0,0 +1,208 @@
import { pick } from '@peertube/peertube-core-utils'
import { FFmpegEdition } from '@peertube/peertube-ffmpeg'
import {
VideoState,
VideoStudioEditionPayload,
VideoStudioTask,
VideoStudioTaskCutPayload,
VideoStudioTaskIntroPayload,
VideoStudioTaskOutroPayload,
VideoStudioTaskPayload,
VideoStudioTaskWatermarkPayload
} from '@peertube/peertube-models'
import { buildUUID } from '@peertube/peertube-node-utils'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
import { CONFIG } from '@server/initializers/config.js'
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
import { isUserQuotaValid } from '@server/lib/user.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo, MVideoFullLight } from '@server/types/models/index.js'
import { MutexInterface } from 'async-mutex'
import { Job } from 'bullmq'
import { remove } from 'fs-extra/esm'
import { extname, join } from 'path'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
const lTagsBase = loggerTagsFactory('video-studio')
async function processVideoStudioEdition (job: Job) {
const payload = job.data as VideoStudioEditionPayload
const lTags = lTagsBase(payload.videoUUID)
logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
try {
const video = await VideoModel.loadFull(payload.videoUUID)
// No video, maybe deleted?
if (!video) {
logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
await safeCleanupStudioTMPFiles(payload.tasks)
return undefined
}
await checkUserQuotaOrThrow(video, payload)
await video.reload()
const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({
videoPath: originalVideoFilePath,
separatedAudioPath
}) => {
let tmpInputFilePath: string
let outputPath: string
for (const task of payload.tasks) {
const outputFilename = buildUUID() + extname(originalVideoFilePath)
outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
await processTask({
videoInputPath: tmpInputFilePath ?? originalVideoFilePath,
separatedAudioInputPath: tmpInputFilePath
? undefined
: separatedAudioPath,
inputFileMutexReleaser,
video,
outputPath,
task,
lTags
})
if (tmpInputFilePath) await remove(tmpInputFilePath)
// For the next iteration
tmpInputFilePath = outputPath
inputFileMutexReleaser = undefined
}
return outputPath
})
logger.info('Video edition ended for video %s.', video.uuid, lTags)
await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })
} catch (err) {
await safeCleanupStudioTMPFiles(payload.tasks)
try {
const video = await VideoModel.loadFull(payload.videoUUID)
await video.setNewState(VideoState.PUBLISHED, false, undefined)
} catch (err) {
logger.error('Cannot reset video state after studio error', { err, ...lTags })
}
throw err
} finally {
if (inputFileMutexReleaser) inputFileMutexReleaser()
}
}
// ---------------------------------------------------------------------------
export {
processVideoStudioEdition
}
// ---------------------------------------------------------------------------
type TaskProcessorOptions<T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = {
videoInputPath: string
separatedAudioInputPath?: string
inputFileMutexReleaser: MutexInterface.Releaser
outputPath: string
video: MVideo
task: T
lTags: { tags: (string | number)[] }
}
const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = {
'add-intro': processAddIntroOutro,
'add-outro': processAddIntroOutro,
'cut': processCut,
'add-watermark': processAddWatermark
}
async function processTask (options: TaskProcessorOptions) {
const { video, task, lTags } = options
logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags })
const processor = taskProcessors[options.task.name]
if (!process) throw new Error('Unknown task ' + task.name)
return processor(options)
}
function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
const { task, lTags } = options
logger.debug('Will add intro/outro to the video.', { options, ...lTags })
return buildFFmpegEdition().addIntroOutro({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
introOutroPath: task.options.file,
type: task.name === 'add-intro'
? 'intro'
: 'outro'
})
}
function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
const { task, lTags } = options
logger.debug('Will cut the video.', { options, ...lTags })
return buildFFmpegEdition().cutVideo({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
start: task.options.start,
end: task.options.end
})
}
function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
const { task, lTags } = options
logger.debug('Will add watermark to the video.', { options, ...lTags })
return buildFFmpegEdition().addWatermark({
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
watermarkPath: task.options.file,
videoFilters: {
watermarkSizeRatio: task.options.watermarkSizeRatio,
horitonzalMarginRatio: task.options.horitonzalMarginRatio,
verticalMarginRatio: task.options.verticalMarginRatio
}
})
}
// ---------------------------------------------------------------------------
async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) {
const user = await UserModel.loadByVideoId(video.id)
const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file
const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder)
if (await isUserQuotaValid({ userId: user.id, uploadSize: additionalBytes }) === false) {
throw new Error('Quota exceeded for this user to edit the video')
}
}
function buildFFmpegEdition () {
return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
}

View File

@@ -0,0 +1,165 @@
import {
HLSTranscodingPayload,
MergeAudioTranscodingPayload,
NewWebVideoResolutionTranscodingPayload,
OptimizeTranscodingPayload,
VideoTranscodingPayload
} from '@peertube/peertube-models'
import { hasMissingHLSStreams } from '@server/lib/runners/job-handlers/shared/utils.js'
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js'
import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding.js'
import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding.js'
import { removeAllWebVideoFiles } from '@server/lib/video-file.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { moveToFailedTranscodingState } from '@server/lib/video-state.js'
import { UserModel } from '@server/models/user/user.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { MUser, MUserId, MVideoFullLight } from '@server/types/models/index.js'
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { VideoModel } from '../../../models/video/video.js'
type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
const handlers: { [id in VideoTranscodingPayload['type']]: HandlerFunction } = {
'new-resolution-to-hls': handleHLSJob,
'new-resolution-to-web-video': handleNewWebVideoResolutionJob,
'merge-audio-to-web-video': handleWebVideoMergeAudioJob,
'optimize-to-web-video': handleWebVideoOptimizeJob
}
const lTags = loggerTagsFactory('transcoding')
async function processVideoTranscoding (job: Job) {
const payload = job.data as VideoTranscodingPayload
logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))
const video = await VideoModel.loadFull(payload.videoUUID)
// No video, maybe deleted?
if (!video) {
logger.info(`Do not process job ${job.id}, video does not exist.`, lTags(payload.videoUUID))
return undefined
}
const user = await UserModel.loadByChannelActorId(video.VideoChannel.Actor.id)
const handler = handlers[payload.type]
if (!handler) {
await moveToFailedTranscodingState(video)
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
throw new Error('Cannot find transcoding handler for ' + payload.type)
}
try {
await handler(job, payload, video, user)
} catch (error) {
await moveToFailedTranscodingState(video)
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
throw error
}
return video
}
// ---------------------------------------------------------------------------
export {
processVideoTranscoding
}
// ---------------------------------------------------------------------------
// Job handlers
// ---------------------------------------------------------------------------
async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
await optimizeOriginalVideofile({ video, job })
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
// ---------------------------------------------------------------------------
async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) {
logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })
logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
// Always move video to next state, we're ready enough with this resolution
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
}
// ---------------------------------------------------------------------------
async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) {
logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
let video: MVideoFullLight
try {
video = await VideoModel.loadFull(videoArg.uuid)
const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
const webVideoFile = video.getWebVideoFileResolution(payload.resolution)
const videoFileInputs = webVideoFile
? [ webVideoFile ]
: [ videoFile, separatedAudioFile ].filter(v => !!v)
await VideoPathManager.Instance.makeAvailableVideoFiles(videoFileInputs, ([ videoPath, separatedAudioPath ]) => {
return generateHlsPlaylistResolution({
video,
videoInputPath: videoPath,
separatedAudioInputPath: separatedAudioPath,
inputFileMutexReleaser,
resolution: payload.resolution,
fps: payload.fps,
separatedAudio: payload.separatedAudio,
job
})
})
} finally {
inputFileMutexReleaser()
}
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
const missingStream = await hasMissingHLSStreams({
inputStreams: payload.inputStreams,
transcodingRequestAt: payload.transcodingRequestAt,
videoId: videoArg.uuid
})
if (!missingStream && payload.deleteWebVideoFiles === true) {
logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid))
await removeAllWebVideoFiles(video)
}
// Splitted audio, wait audio & video generation before moving the video in its next state
const moveVideoToNextState = payload.canMoveVideoState && !missingStream
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState, video })
}

View File

@@ -0,0 +1,21 @@
import { VideoTranscriptionPayload } from '@peertube/peertube-models'
import { generateSubtitle } from '@server/lib/video-captions.js'
import { Job } from 'bullmq'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { VideoModel } from '../../../models/video/video.js'
const lTags = loggerTagsFactory('transcription')
export async function processVideoTranscription (job: Job) {
const payload = job.data as VideoTranscriptionPayload
logger.info('Processing video transcription in job %s.', job.id)
const video = await VideoModel.load(payload.videoUUID)
if (!video) {
logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))
return
}
return generateSubtitle({ video })
}

View File

@@ -0,0 +1,57 @@
import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
import { VideoViewModel } from '@server/models/view/video-view.js'
import { logger } from '../../../helpers/logger.js'
import { VideoModel } from '../../../models/video/video.js'
import { Redis } from '../../redis.js'
async function processVideosViewsStats () {
const lastHour = new Date()
// In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour
if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1)
const hour = lastHour.getHours()
const startDate = lastHour.setMinutes(0, 0, 0)
const endDate = lastHour.setMinutes(59, 59, 999)
const videoIds = await Redis.Instance.listVideosViewedForStats(hour)
if (videoIds.length === 0) return
logger.info('Processing videos views stats in job for hour %d.', hour)
for (const videoId of videoIds) {
try {
const views = await Redis.Instance.getVideoViewsStats(videoId, hour)
await Redis.Instance.deleteVideoViewsStats(videoId, hour)
if (views) {
logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour)
try {
const video = await VideoModel.load(videoId)
if (!video) {
logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId)
continue
}
await VideoViewModel.create({
startDate: new Date(startDate),
endDate: new Date(endDate),
views,
videoId
})
} catch (err) {
logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err })
}
}
} catch (err) {
logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err })
}
}
}
// ---------------------------------------------------------------------------
export {
processVideosViewsStats
}