Init commit
This commit is contained in:
202
server/core/lib/job-queue/handlers/activitypub-cleaner.ts
Normal file
202
server/core/lib/job-queue/handlers/activitypub-cleaner.ts
Normal file
@@ -0,0 +1,202 @@
|
||||
import Bluebird from 'bluebird'
|
||||
import { Job } from 'bullmq'
|
||||
import {
|
||||
isAnnounceActivityValid,
|
||||
isDislikeActivityValid,
|
||||
isLikeActivityValid
|
||||
} from '@server/helpers/custom-validators/activitypub/activity.js'
|
||||
import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments.js'
|
||||
import { PeerTubeRequestError } from '@server/helpers/requests.js'
|
||||
import { AP_CLEANER } from '@server/initializers/constants.js'
|
||||
import { fetchAP } from '@server/lib/activitypub/activity.js'
|
||||
import { checkUrlsSameHost } from '@server/lib/activitypub/url.js'
|
||||
import { Redis } from '@server/lib/redis.js'
|
||||
import { VideoCommentModel } from '@server/models/video/video-comment.js'
|
||||
import { VideoShareModel } from '@server/models/video/video-share.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { HttpStatusCode } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js'
|
||||
|
||||
const lTags = loggerTagsFactory('ap-cleaner')
|
||||
|
||||
// Job to clean remote interactions off local videos
|
||||
|
||||
async function processActivityPubCleaner (_job: Job) {
|
||||
logger.info('Processing ActivityPub cleaner.', lTags())
|
||||
|
||||
{
|
||||
const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
|
||||
const { bodyValidator, deleter, updater } = rateOptionsFactory()
|
||||
|
||||
await Bluebird.map(rateUrls, async rateUrl => {
|
||||
// TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed
|
||||
if (rateUrl.includes('#')) return
|
||||
|
||||
const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
|
||||
|
||||
if (result?.status === 'deleted') {
|
||||
const { videoId, type } = result.data
|
||||
|
||||
await VideoModel.syncLocalRates(videoId, type, undefined)
|
||||
}
|
||||
}, { concurrency: AP_CLEANER.CONCURRENCY })
|
||||
}
|
||||
|
||||
{
|
||||
const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos()
|
||||
const { bodyValidator, deleter, updater } = shareOptionsFactory()
|
||||
|
||||
await Bluebird.map(shareUrls, async shareUrl => {
|
||||
await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
|
||||
}, { concurrency: AP_CLEANER.CONCURRENCY })
|
||||
}
|
||||
|
||||
{
|
||||
const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos()
|
||||
const { bodyValidator, deleter, updater } = commentOptionsFactory()
|
||||
|
||||
await Bluebird.map(commentUrls, async commentUrl => {
|
||||
await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
|
||||
}, { concurrency: AP_CLEANER.CONCURRENCY })
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubCleaner
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function updateObjectIfNeeded <T> (options: {
|
||||
url: string
|
||||
bodyValidator: (body: any) => boolean
|
||||
updater: (url: string, newUrl: string) => Promise<T>
|
||||
deleter: (url: string) => Promise<T> }
|
||||
): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
|
||||
const { url, bodyValidator, updater, deleter } = options
|
||||
|
||||
const on404OrTombstone = async () => {
|
||||
logger.info('Removing remote AP object %s.', url, lTags(url))
|
||||
const data = await deleter(url)
|
||||
|
||||
return { status: 'deleted' as 'deleted', data }
|
||||
}
|
||||
|
||||
try {
|
||||
const { body } = await fetchAP<any>(url)
|
||||
|
||||
// If not same id, check same host and update
|
||||
if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
|
||||
|
||||
if (body.type === 'Tombstone') {
|
||||
return on404OrTombstone()
|
||||
}
|
||||
|
||||
const newUrl = body.id
|
||||
if (newUrl !== url) {
|
||||
if (checkUrlsSameHost(newUrl, url) !== true) {
|
||||
throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
|
||||
}
|
||||
|
||||
logger.info('Updating remote AP object %s.', url, lTags(url))
|
||||
const data = await updater(url, newUrl)
|
||||
|
||||
return { status: 'updated', data }
|
||||
}
|
||||
|
||||
return null
|
||||
} catch (err) {
|
||||
// Does not exist anymore, remove entry
|
||||
if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
|
||||
return on404OrTombstone()
|
||||
}
|
||||
|
||||
logger.debug('Remote AP object %s is unavailable.', url, lTags(url))
|
||||
|
||||
const unavailability = await Redis.Instance.addAPUnavailability(url)
|
||||
if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) {
|
||||
logger.info('Removing unavailable AP resource %s.', url, lTags(url))
|
||||
return on404OrTombstone()
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function rateOptionsFactory () {
|
||||
return {
|
||||
bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body),
|
||||
|
||||
updater: async (url: string, newUrl: string) => {
|
||||
const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
|
||||
rate.url = newUrl
|
||||
|
||||
const videoId = rate.videoId
|
||||
const type = rate.type
|
||||
|
||||
await rate.save()
|
||||
|
||||
return { videoId, type }
|
||||
},
|
||||
|
||||
deleter: async (url) => {
|
||||
const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
|
||||
|
||||
const videoId = rate.videoId
|
||||
const type = rate.type
|
||||
|
||||
await rate.destroy()
|
||||
|
||||
return { videoId, type }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function shareOptionsFactory () {
|
||||
return {
|
||||
bodyValidator: (body: any) => isAnnounceActivityValid(body),
|
||||
|
||||
updater: async (url: string, newUrl: string) => {
|
||||
const share = await VideoShareModel.loadByUrl(url, undefined)
|
||||
share.url = newUrl
|
||||
|
||||
await share.save()
|
||||
|
||||
return undefined
|
||||
},
|
||||
|
||||
deleter: async (url) => {
|
||||
const share = await VideoShareModel.loadByUrl(url, undefined)
|
||||
|
||||
await share.destroy()
|
||||
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function commentOptionsFactory () {
|
||||
return {
|
||||
bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body),
|
||||
|
||||
updater: async (url: string, newUrl: string) => {
|
||||
const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(url)
|
||||
comment.url = newUrl
|
||||
|
||||
await comment.save()
|
||||
|
||||
return undefined
|
||||
},
|
||||
|
||||
deleter: async (url) => {
|
||||
const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideoAndReply(url)
|
||||
|
||||
await comment.destroy()
|
||||
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
103
server/core/lib/job-queue/handlers/activitypub-follow.ts
Normal file
103
server/core/lib/job-queue/handlers/activitypub-follow.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js'
|
||||
import { ActivitypubFollowPayload } from '@peertube/peertube-models'
|
||||
import { sanitizeHost } from '../../../helpers/core-utils.js'
|
||||
import { retryTransactionWrapper } from '../../../helpers/database-utils.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { REMOTE_SCHEME, SERVER_ACTOR_NAME, WEBSERVER } from '../../../initializers/constants.js'
|
||||
import { sequelizeTypescript } from '../../../initializers/database.js'
|
||||
import { ActorModel } from '../../../models/actor/actor.js'
|
||||
import { ActorFollowModel } from '../../../models/actor/actor-follow.js'
|
||||
import { MActor, MActorFull } from '../../../types/models/index.js'
|
||||
import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors/index.js'
|
||||
import { sendFollow } from '../../activitypub/send/index.js'
|
||||
import { Notifier } from '../../notifier/index.js'
|
||||
import { getApplicationActorOfHost } from '@server/helpers/activity-pub-utils.js'
|
||||
|
||||
async function processActivityPubFollow (job: Job) {
|
||||
const payload = job.data as ActivitypubFollowPayload
|
||||
const host = payload.host
|
||||
|
||||
const identifier = [ payload.name || SERVER_ACTOR_NAME, host ]
|
||||
.filter(v => !!v).join('@')
|
||||
|
||||
logger.info('Processing ActivityPub follow in job %s.', job.id)
|
||||
|
||||
let targetActor: MActorFull
|
||||
if (!host || host === WEBSERVER.HOST) {
|
||||
if (!payload.name) throw new Error('Payload name is mandatory for local follow')
|
||||
|
||||
targetActor = await ActorModel.loadLocalByName(payload.name)
|
||||
} else {
|
||||
const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP)
|
||||
|
||||
let actorUrl: string
|
||||
|
||||
try {
|
||||
if (!payload.name) actorUrl = await getApplicationActorOfHost(sanitizedHost)
|
||||
if (!actorUrl) actorUrl = await loadActorUrlOrGetFromWebfinger((payload.name || SERVER_ACTOR_NAME) + '@' + sanitizedHost)
|
||||
|
||||
targetActor = await getOrCreateAPActor(actorUrl, 'all')
|
||||
} catch (err) {
|
||||
logger.warn(`Do not follow ${identifier} because we could not find the actor URL (in database or using webfinger)`, { err })
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if (!targetActor) {
|
||||
logger.warn(`Do not follow ${identifier} because we could not fetch/load the actor`)
|
||||
return
|
||||
}
|
||||
|
||||
if (payload.assertIsChannel && !targetActor.VideoChannel) {
|
||||
logger.warn(`Do not follow ${identifier} because it is not a channel.`)
|
||||
return
|
||||
}
|
||||
|
||||
const fromActor = await ActorModel.load(payload.followerActorId)
|
||||
|
||||
return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow)
|
||||
}
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubFollow
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) {
|
||||
if (fromActor.id === targetActor.id) {
|
||||
throw new Error('Follower is the same as target actor.')
|
||||
}
|
||||
|
||||
// Same server, direct accept
|
||||
const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending'
|
||||
|
||||
const actorFollow = await sequelizeTypescript.transaction(async t => {
|
||||
const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({
|
||||
byActor: fromActor,
|
||||
state,
|
||||
targetActor,
|
||||
activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor),
|
||||
transaction: t
|
||||
})
|
||||
|
||||
// Send a notification to remote server if our follow is not already accepted
|
||||
if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t)
|
||||
|
||||
return actorFollow
|
||||
})
|
||||
|
||||
const followerFull = await ActorModel.loadFull(fromActor.id)
|
||||
|
||||
const actorFollowFull = Object.assign(actorFollow, {
|
||||
ActorFollowing: targetActor,
|
||||
ActorFollower: followerFull
|
||||
})
|
||||
|
||||
if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull)
|
||||
if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull)
|
||||
|
||||
return actorFollow
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { ActivitypubHttpBroadcastPayload } from '@peertube/peertube-models'
|
||||
import { buildGlobalHTTPHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/http.js'
|
||||
import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache.js'
|
||||
import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
// Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive
|
||||
|
||||
async function processActivityPubHttpSequentialBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) {
|
||||
logger.info('Processing ActivityPub broadcast in job %s.', job.id)
|
||||
|
||||
const requestOptions = await buildRequestOptions(job.data)
|
||||
|
||||
const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
|
||||
|
||||
return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
|
||||
}
|
||||
|
||||
async function processActivityPubParallelHttpBroadcast (job: Job<ActivitypubHttpBroadcastPayload>) {
|
||||
logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id)
|
||||
|
||||
const requestOptions = await buildRequestOptions(job.data)
|
||||
|
||||
const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions })
|
||||
|
||||
return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubHttpSequentialBroadcast,
|
||||
processActivityPubParallelHttpBroadcast
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) {
|
||||
const body = await computeBody(payload)
|
||||
const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true })
|
||||
|
||||
return {
|
||||
method: 'POST' as 'POST',
|
||||
json: body,
|
||||
httpSignature: httpSignatureOptions,
|
||||
headers: await buildGlobalHTTPHeaders(body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { ActivitypubHttpFetcherPayload, FetchType } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
import { VideoCommentModel } from '../../../models/video/video-comment.js'
|
||||
import { VideoShareModel } from '../../../models/video/video-share.js'
|
||||
import { MAccountDefault, MVideoFullLight } from '../../../types/models/index.js'
|
||||
import { crawlCollectionPage } from '../../activitypub/crawl.js'
|
||||
import { createAccountPlaylists } from '../../activitypub/playlists/index.js'
|
||||
import { processActivities } from '../../activitypub/process/index.js'
|
||||
import { addVideoShares } from '../../activitypub/share.js'
|
||||
import { addVideoComments } from '../../activitypub/video-comments.js'
|
||||
import { AccountModel } from '@server/models/account/account.js'
|
||||
|
||||
async function processActivityPubHttpFetcher (job: Job) {
|
||||
logger.info('Processing ActivityPub fetcher in job %s.', job.id)
|
||||
|
||||
const payload = job.data as ActivitypubHttpFetcherPayload
|
||||
|
||||
let video: MVideoFullLight
|
||||
if (payload.videoId) video = await VideoModel.loadFull(payload.videoId)
|
||||
|
||||
let account: MAccountDefault
|
||||
if (payload.accountId) account = await AccountModel.load(payload.accountId)
|
||||
|
||||
const fetcherType: { [id in FetchType]: (items: any[]) => Promise<any> } = {
|
||||
'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
|
||||
'video-shares': items => addVideoShares(items, video),
|
||||
'video-comments': items => addVideoComments(items),
|
||||
'account-playlists': items => createAccountPlaylists(items, account)
|
||||
}
|
||||
|
||||
const cleanerType: { [id in FetchType]?: (crawlStartDate: Date) => Promise<any> } = {
|
||||
'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
|
||||
'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
|
||||
}
|
||||
|
||||
return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubHttpFetcher
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { ActivitypubHttpUnicastPayload } from '@peertube/peertube-models'
|
||||
import { buildGlobalHTTPHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/http.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { ActorFollowHealthCache } from '../../actor-follow-health-cache.js'
|
||||
import { httpUnicastFromWorker } from '@server/lib/worker/parent-process.js'
|
||||
|
||||
async function processActivityPubHttpUnicast (job: Job) {
|
||||
logger.info('Processing ActivityPub unicast in job %s.', job.id)
|
||||
|
||||
const payload = job.data as ActivitypubHttpUnicastPayload
|
||||
const uri = payload.uri
|
||||
|
||||
const body = await computeBody(payload)
|
||||
const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true })
|
||||
|
||||
const options = {
|
||||
method: 'POST' as 'POST',
|
||||
json: body,
|
||||
httpSignature: httpSignatureOptions,
|
||||
headers: await buildGlobalHTTPHeaders(body)
|
||||
}
|
||||
|
||||
try {
|
||||
await httpUnicastFromWorker({ uri, requestOptions: options })
|
||||
ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], [])
|
||||
} catch (err) {
|
||||
ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ])
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActivityPubHttpUnicast
|
||||
}
|
||||
60
server/core/lib/job-queue/handlers/activitypub-refresher.ts
Normal file
60
server/core/lib/job-queue/handlers/activitypub-refresher.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import { RefreshPayload } from '@peertube/peertube-models'
|
||||
import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists/index.js'
|
||||
import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders/index.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { ActorModel } from '../../../models/actor/actor.js'
|
||||
import { VideoPlaylistModel } from '../../../models/video/video-playlist.js'
|
||||
import { refreshActorIfNeeded } from '../../activitypub/actors/index.js'
|
||||
|
||||
async function refreshAPObject (job: Job) {
|
||||
const payload = job.data as RefreshPayload
|
||||
|
||||
logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url)
|
||||
|
||||
if (payload.type === 'video') return refreshVideo(payload.url)
|
||||
if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
|
||||
if (payload.type === 'actor') return refreshActor(payload.url)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
refreshAPObject
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function refreshVideo (videoUrl: string) {
|
||||
const fetchType = 'all'
|
||||
const syncParam = { rates: true, shares: true, comments: true }
|
||||
|
||||
const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType)
|
||||
if (videoFromDatabase) {
|
||||
const refreshOptions = {
|
||||
video: videoFromDatabase,
|
||||
fetchedType: fetchType as VideoLoadByUrlType,
|
||||
syncParam
|
||||
}
|
||||
|
||||
await refreshVideoIfNeeded(refreshOptions)
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshActor (actorUrl: string) {
|
||||
const fetchType = 'all'
|
||||
const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl)
|
||||
|
||||
if (actor) {
|
||||
await refreshActorIfNeeded({ actor, fetchedType: fetchType })
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshVideoPlaylist (playlistUrl: string) {
|
||||
const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
|
||||
|
||||
if (playlist) {
|
||||
await refreshVideoPlaylistIfNeeded(playlist)
|
||||
}
|
||||
}
|
||||
20
server/core/lib/job-queue/handlers/actor-keys.ts
Normal file
20
server/core/lib/job-queue/handlers/actor-keys.ts
Normal file
@@ -0,0 +1,20 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors/index.js'
|
||||
import { ActorModel } from '@server/models/actor/actor.js'
|
||||
import { ActorKeysPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processActorKeys (job: Job) {
|
||||
const payload = job.data as ActorKeysPayload
|
||||
logger.info('Processing actor keys in job %s.', job.id)
|
||||
|
||||
const actor = await ActorModel.load(payload.actorId)
|
||||
|
||||
await generateAndSaveActorKeys(actor)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processActorKeys
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
|
||||
import { Job } from 'bullmq'
|
||||
|
||||
const lTags = loggerTagsFactory('channel-synchronization')
|
||||
|
||||
export async function processAfterVideoChannelImport (job: Job) {
|
||||
const payload = job.data as AfterVideoChannelImportPayload
|
||||
if (!payload.channelSyncId) return
|
||||
|
||||
logger.info('Processing after video channel import in job %s.', job.id)
|
||||
|
||||
const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId)
|
||||
if (!sync) {
|
||||
logger.error('Unknown sync id %d.', payload.channelSyncId)
|
||||
return
|
||||
}
|
||||
|
||||
const childrenValues = await job.getChildrenValues<VideoImportPreventExceptionResult>()
|
||||
|
||||
let errors = payload.buildJobErrors || 0
|
||||
let successes = 0
|
||||
|
||||
for (const value of Object.values(childrenValues)) {
|
||||
if (value.resultType === 'success') successes++
|
||||
else if (value.resultType === 'error') errors++
|
||||
}
|
||||
|
||||
if (errors > 0) {
|
||||
sync.state = VideoChannelSyncState.FAILED
|
||||
logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes, ...lTags() })
|
||||
} else {
|
||||
sync.state = VideoChannelSyncState.SYNCED
|
||||
logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes, ...lTags() })
|
||||
}
|
||||
|
||||
await sync.save()
|
||||
}
|
||||
34
server/core/lib/job-queue/handlers/create-user-export.ts
Normal file
34
server/core/lib/job-queue/handlers/create-user-export.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { CreateUserExportPayload } from '@peertube/peertube-models'
|
||||
import { UserExportModel } from '@server/models/user/user-export.js'
|
||||
import { UserExporter } from '@server/lib/user-import-export/user-exporter.js'
|
||||
import { Emailer } from '@server/lib/emailer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-export')
|
||||
|
||||
export async function processCreateUserExport (job: Job): Promise<void> {
|
||||
const payload = job.data as CreateUserExportPayload
|
||||
const exportModel = await UserExportModel.load(payload.userExportId)
|
||||
|
||||
logger.info('Processing create user export %s in job %s.', payload.userExportId, job.id, lTags())
|
||||
|
||||
if (!exportModel) {
|
||||
logger.info(`User export ${payload.userExportId} does not exist anymore, do not create user export.`, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
const exporter = new UserExporter()
|
||||
|
||||
try {
|
||||
await exporter.export(exportModel)
|
||||
|
||||
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
|
||||
|
||||
logger.info(`User export ${payload.userExportId} has been created`, lTags())
|
||||
} catch (err) {
|
||||
await Emailer.Instance.addUserExportCompletedOrErroredJob(exportModel)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
11
server/core/lib/job-queue/handlers/email.ts
Normal file
11
server/core/lib/job-queue/handlers/email.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { EmailPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { Emailer } from '../../emailer.js'
|
||||
|
||||
export function processEmail (job: Job) {
|
||||
const payload = job.data as EmailPayload
|
||||
logger.info('Processing email in job %s.', job.id)
|
||||
|
||||
return Emailer.Instance.sendMail(payload)
|
||||
}
|
||||
28
server/core/lib/job-queue/handlers/federate-video.ts
Normal file
28
server/core/lib/job-queue/handlers/federate-video.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
|
||||
import { sequelizeTypescript } from '@server/initializers/database.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { FederateVideoPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
function processFederateVideo (job: Job) {
|
||||
const payload = job.data as FederateVideoPayload
|
||||
|
||||
logger.info('Processing video federation in job %s.', job.id)
|
||||
|
||||
return retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID, t)
|
||||
if (!video) return
|
||||
|
||||
return federateVideoIfNeeded(video, payload.isNewVideoForFederation, t)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processFederateVideo
|
||||
}
|
||||
100
server/core/lib/job-queue/handlers/generate-storyboard.ts
Normal file
100
server/core/lib/job-queue/handlers/generate-storyboard.ts
Normal file
@@ -0,0 +1,100 @@
|
||||
import { FFmpegImage } from '@peertube/peertube-ffmpeg'
|
||||
import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
|
||||
import { generateImageFilename } from '@server/helpers/image-utils.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { STORYBOARD } from '@server/initializers/constants.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { join } from 'path'
|
||||
import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('storyboard')
|
||||
|
||||
export async function processGenerateStoryboard (job: Job): Promise<void> {
|
||||
const payload = job.data as GenerateStoryboardPayload
|
||||
const lTags = lTagsBase(payload.videoUUID)
|
||||
|
||||
logger.info(`Processing generate storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
|
||||
|
||||
if (CONFIG.STORYBOARDS.ENABLED !== true) {
|
||||
logger.info(`Storyboard disabled, do not process storyboard of ${payload.videoUUID} in job ${job.id}.`, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
if (!video) {
|
||||
logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
|
||||
if (!inputFile) {
|
||||
logger.info(`Do not generate a storyboard of ${payload.videoUUID} since the video does not have a video stream`, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
|
||||
const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
|
||||
|
||||
const { totalSprites, spriteDuration } = buildTotalSprites(video)
|
||||
if (totalSprites === 0) {
|
||||
logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)
|
||||
return
|
||||
}
|
||||
|
||||
const spritesCount = findGridSize({
|
||||
toFind: totalSprites,
|
||||
maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
|
||||
})
|
||||
|
||||
const filename = generateImageFilename()
|
||||
const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
|
||||
|
||||
logger.debug(
|
||||
`Generating storyboard from video of ${video.uuid} to ${destination}`,
|
||||
{ ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
|
||||
)
|
||||
|
||||
const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
|
||||
|
||||
await ffmpeg.generateStoryboardFromVideo({
|
||||
destination,
|
||||
path: videoPath,
|
||||
inputFileMutexReleaser,
|
||||
sprites: {
|
||||
size: {
|
||||
height: spriteHeight,
|
||||
width: spriteWidth
|
||||
},
|
||||
count: spritesCount,
|
||||
duration: spriteDuration
|
||||
}
|
||||
})
|
||||
|
||||
await insertStoryboardInDatabase({
|
||||
videoUUID: video.uuid,
|
||||
lTags,
|
||||
|
||||
filename,
|
||||
destination,
|
||||
|
||||
imageSize: await getImageSizeFromWorker(destination),
|
||||
|
||||
spriteHeight,
|
||||
spriteWidth,
|
||||
spriteDuration,
|
||||
|
||||
federate: payload.federate
|
||||
})
|
||||
})
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
33
server/core/lib/job-queue/handlers/import-user-archive.ts
Normal file
33
server/core/lib/job-queue/handlers/import-user-archive.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { ImportUserArchivePayload } from '@peertube/peertube-models'
|
||||
import { UserImportModel } from '@server/models/user/user-import.js'
|
||||
import { UserImporter } from '@server/lib/user-import-export/user-importer.js'
|
||||
import { Emailer } from '@server/lib/emailer.js'
|
||||
|
||||
const lTags = loggerTagsFactory('user-import')
|
||||
|
||||
export async function processImportUserArchive (job: Job): Promise<void> {
|
||||
const payload = job.data as ImportUserArchivePayload
|
||||
const importModel = await UserImportModel.load(payload.userImportId)
|
||||
|
||||
logger.info(`Processing importing user archive ${payload.userImportId} in job ${job.id}`, lTags())
|
||||
|
||||
if (!importModel) {
|
||||
logger.info(`User import ${payload.userImportId} does not exist anymore, do not create import data.`, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
const exporter = new UserImporter()
|
||||
await exporter.import(importModel)
|
||||
|
||||
try {
|
||||
await Emailer.Instance.addUserImportSuccessJob(importModel)
|
||||
|
||||
logger.info(`User import ${payload.userImportId} ended`, lTags())
|
||||
} catch (err) {
|
||||
await Emailer.Instance.addUserImportErroredJob(importModel)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
64
server/core/lib/job-queue/handlers/live-video-start.ts
Normal file
64
server/core/lib/job-queue/handlers/live-video-start.ts
Normal file
@@ -0,0 +1,64 @@
|
||||
import { LiveVideoStartPayload } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { startScheduledLiveSession } from '@server/controllers/api/live-stream.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { LiveVideoProcessManager } from '@server/lib/live-videos/live-video-process-manager.js'
|
||||
import { LIVE_VIDEO_START_LEAD_TIME_MS } from '@server/lib/live-videos/shared.js'
|
||||
import { LiveVideoModel, LiveVideoStatus } from '@server/models/video/live-video.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { setTimeout as wait } from 'node:timers/promises'
|
||||
|
||||
const lTags = loggerTagsFactory('jobs', 'live-video-start')
|
||||
const LIVE_VIDEO_START_POLL_MS = 5_000
|
||||
|
||||
export async function processLiveVideoStart (job: Job) {
|
||||
const payload = job.data as LiveVideoStartPayload
|
||||
let liveVideo: LiveVideoModel
|
||||
|
||||
while (true) {
|
||||
const currentLiveVideo = await LiveVideoModel.loadById(payload.liveVideoId)
|
||||
if (!currentLiveVideo) {
|
||||
logger.warn('Cannot find live video %d for scheduled start job %s.', payload.liveVideoId, job.id, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
if (currentLiveVideo.status !== LiveVideoStatus.SCHEDULED) {
|
||||
logger.info('Skipping scheduled start of live video %d because status is %s.', currentLiveVideo.id, currentLiveVideo.status, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
const now = Date.now()
|
||||
const startAt = currentLiveVideo.startTime.getTime()
|
||||
const endAt = currentLiveVideo.endTime.getTime()
|
||||
if (endAt <= now) {
|
||||
logger.info('Skipping scheduled start of live video %d because it is already finished.', currentLiveVideo.id, lTags())
|
||||
return
|
||||
}
|
||||
|
||||
const launchAt = startAt - LIVE_VIDEO_START_LEAD_TIME_MS
|
||||
const waitMs = launchAt - now
|
||||
if (waitMs <= 0) {
|
||||
liveVideo = currentLiveVideo
|
||||
break
|
||||
}
|
||||
|
||||
await wait(Math.min(waitMs, LIVE_VIDEO_START_POLL_MS))
|
||||
}
|
||||
|
||||
if (CONFIG.LIVE.ENABLE_SOURCE_START) {
|
||||
await LiveVideoProcessManager.Instance.start(liveVideo)
|
||||
} else {
|
||||
logger.info('Skipping source-side scheduled live start for live video %d because live.enable_source_start is disabled.', liveVideo.id, lTags())
|
||||
}
|
||||
|
||||
const sinkWaitMs = liveVideo.startTime.getTime() - Date.now()
|
||||
if (sinkWaitMs > 0) {
|
||||
await wait(sinkWaitMs)
|
||||
}
|
||||
|
||||
if (CONFIG.LIVE.ENABLE_SINK_START) {
|
||||
await startScheduledLiveSession(liveVideo.targetCamera, { keepAliveUntil: liveVideo.endTime })
|
||||
} else {
|
||||
logger.info('Skipping sink-side scheduled live start for live video %d because live.enable_sink_start is disabled.', liveVideo.id, lTags())
|
||||
}
|
||||
}
|
||||
110
server/core/lib/job-queue/handlers/manage-video-torrent.ts
Normal file
110
server/core/lib/job-queue/handlers/manage-video-torrent.ts
Normal file
@@ -0,0 +1,110 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { extractVideo } from '@server/helpers/video.js'
|
||||
import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/lib/webtorrent.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import { ManageVideoTorrentPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processManageVideoTorrent (job: Job) {
|
||||
const payload = job.data as ManageVideoTorrentPayload
|
||||
logger.info('Processing torrent in job %s.', job.id)
|
||||
|
||||
if (payload.action === 'create') return doCreateAction(payload)
|
||||
if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processManageVideoTorrent
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) {
|
||||
const [ video, file ] = await Promise.all([
|
||||
loadVideoOrLog(payload.videoId),
|
||||
loadFileOrLog(payload.videoFileId)
|
||||
])
|
||||
|
||||
if (!video || !file) return
|
||||
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
await file.reload()
|
||||
|
||||
await createTorrentAndSetInfoHash(video, file)
|
||||
|
||||
// Refresh videoFile because the createTorrentAndSetInfoHash could be long
|
||||
const refreshedFile = await VideoFileModel.loadWithVideo(file.id)
|
||||
// File does not exist anymore, remove the generated torrent
|
||||
if (!refreshedFile) return file.removeTorrent()
|
||||
|
||||
refreshedFile.infoHash = file.infoHash
|
||||
refreshedFile.torrentFilename = file.torrentFilename
|
||||
|
||||
await refreshedFile.save()
|
||||
} finally {
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) {
|
||||
const [ video, streamingPlaylist, file ] = await Promise.all([
|
||||
loadVideoOrLog(payload.videoId),
|
||||
loadStreamingPlaylistOrLog(payload.streamingPlaylistId),
|
||||
loadFileOrLog(payload.videoFileId)
|
||||
])
|
||||
|
||||
if ((!video && !streamingPlaylist) || !file) return
|
||||
|
||||
const extractedVideo = extractVideo(video || streamingPlaylist)
|
||||
const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid)
|
||||
|
||||
try {
|
||||
await updateTorrentMetadata(video || streamingPlaylist, file)
|
||||
|
||||
await file.save()
|
||||
} finally {
|
||||
fileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
async function loadVideoOrLog (videoId: number) {
|
||||
if (!videoId) return undefined
|
||||
|
||||
const video = await VideoModel.load(videoId)
|
||||
if (!video) {
|
||||
logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId)
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) {
|
||||
if (!streamingPlaylistId) return undefined
|
||||
|
||||
const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId)
|
||||
if (!streamingPlaylist) {
|
||||
logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId)
|
||||
}
|
||||
|
||||
return streamingPlaylist
|
||||
}
|
||||
|
||||
async function loadFileOrLog (videoFileId: number) {
|
||||
if (!videoFileId) return undefined
|
||||
|
||||
const file = await VideoFileModel.load(videoFileId)
|
||||
|
||||
if (!file) {
|
||||
logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId)
|
||||
}
|
||||
|
||||
return file
|
||||
}
|
||||
45
server/core/lib/job-queue/handlers/move-to-file-system.ts
Normal file
45
server/core/lib/job-queue/handlers/move-to-file-system.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
import { isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { moveCaptionToFS, moveVideoToFS, onMoveVideoToFSFailure } from '@server/lib/move-storage/move-to-file-system.js'
|
||||
import { Job } from 'bullmq'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-file-system')
|
||||
|
||||
export async function processMoveToFileSystem (job: Job) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
|
||||
if (isMoveVideoStoragePayload(payload)) { // Move all video related files
|
||||
logger.info(`Moving video ${payload.videoUUID} to file system in job ${job.id}.`, lTagsBase(payload.videoUUID))
|
||||
|
||||
const moveVideoState = payload.isNewVideo !== undefined
|
||||
? { isNewVideo: payload.isNewVideo, previousVideoState: payload.previousVideoState }
|
||||
: payload.moveVideoState
|
||||
|
||||
return moveVideoToFS({
|
||||
videoUUID: payload.videoUUID,
|
||||
moveVideoState,
|
||||
loggerTags: lTagsBase().tags
|
||||
})
|
||||
} else if (isMoveCaptionPayload(payload)) { // Only caption file
|
||||
logger.info(`Moving video caption ${payload.captionId} to file system in job ${job.id}.`, lTagsBase(payload.captionId))
|
||||
|
||||
return moveCaptionToFS({
|
||||
captionId: payload.captionId,
|
||||
loggerTags: lTagsBase().tags
|
||||
})
|
||||
} else {
|
||||
throw new Error('Unknown payload type')
|
||||
}
|
||||
}
|
||||
|
||||
export async function onMoveToFileSystemFailure (job: Job, err: any) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
|
||||
if (!isMoveVideoStoragePayload(payload)) return
|
||||
|
||||
await onMoveVideoToFSFailure({
|
||||
videoUUID: payload.videoUUID,
|
||||
err,
|
||||
loggerTags: lTagsBase().tags
|
||||
})
|
||||
}
|
||||
49
server/core/lib/job-queue/handlers/move-to-object-storage.ts
Normal file
49
server/core/lib/job-queue/handlers/move-to-object-storage.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { isMoveCaptionPayload, isMoveVideoStoragePayload, MoveStoragePayload } from '@peertube/peertube-models'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import {
|
||||
moveCaptionToObjectStorage,
|
||||
moveVideoToObjectStorage,
|
||||
onMoveVideoToObjectStorageFailure
|
||||
} from '@server/lib/move-storage/move-to-object-storage.js'
|
||||
import { Job } from 'bullmq'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('move-object-storage')
|
||||
|
||||
export async function processMoveToObjectStorage (job: Job) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
|
||||
if (isMoveVideoStoragePayload(payload)) { // Move all video related files
|
||||
logger.info(`Moving video ${payload.videoUUID} to object storage in job ${job.id}`, lTagsBase(payload.videoUUID))
|
||||
|
||||
const moveVideoState = payload.isNewVideo !== undefined
|
||||
? { isNewVideo: payload.isNewVideo, previousVideoState: payload.previousVideoState }
|
||||
: payload.moveVideoState
|
||||
|
||||
await moveVideoToObjectStorage({
|
||||
videoUUID: payload.videoUUID,
|
||||
moveVideoState,
|
||||
loggerTags: lTagsBase().tags
|
||||
})
|
||||
} else if (isMoveCaptionPayload(payload)) { // Only caption file
|
||||
logger.info(`Moving video caption ${payload.captionId} to object storage in job ${job.id}.`, lTagsBase(payload.captionId))
|
||||
|
||||
return moveCaptionToObjectStorage({
|
||||
captionId: payload.captionId,
|
||||
loggerTags: lTagsBase().tags
|
||||
})
|
||||
} else {
|
||||
throw new Error('Unknown payload type')
|
||||
}
|
||||
}
|
||||
|
||||
export async function onMoveToObjectStorageFailure (job: Job, err: any) {
|
||||
const payload = job.data as MoveStoragePayload
|
||||
|
||||
if (!isMoveVideoStoragePayload(payload)) return
|
||||
|
||||
await onMoveVideoToObjectStorageFailure({
|
||||
videoUUID: payload.videoUUID,
|
||||
err,
|
||||
loggerTags: lTagsBase().tags
|
||||
})
|
||||
}
|
||||
27
server/core/lib/job-queue/handlers/notify.ts
Normal file
27
server/core/lib/job-queue/handlers/notify.ts
Normal file
@@ -0,0 +1,27 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { Notifier } from '@server/lib/notifier/index.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { NotifyPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processNotify (job: Job) {
|
||||
const payload = job.data as NotifyPayload
|
||||
logger.info('Processing %s notification in job %s.', payload.action, job.id)
|
||||
|
||||
if (payload.action === 'new-video') return doNotifyNewVideo(payload)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processNotify
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) {
|
||||
const refreshedVideo = await VideoModel.loadFull(payload.videoUUID)
|
||||
if (!refreshedVideo) return
|
||||
|
||||
Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(refreshedVideo)
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { TranscodingJobBuilderPayload, VideoFileStream } from '@peertube/peertube-models'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
|
||||
/**
|
||||
* Create transcoding jobs.
|
||||
* They are not created directly so we can keep parent/children relationships using bullmq features
|
||||
* while having the ability to use remote runner jobs that are not part of bullmq
|
||||
*/
|
||||
|
||||
async function processTranscodingJobBuilder (job: Job) {
|
||||
const payload = job.data as TranscodingJobBuilderPayload
|
||||
|
||||
logger.info('Processing transcoding job builder in job %s.', job.id)
|
||||
|
||||
if (payload.optimizeJob) {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
const user = await UserModel.loadByVideoId(video.id)
|
||||
const videoFile = video.getMaxQualityFile(VideoFileStream.VIDEO) || video.getMaxQualityFile(VideoFileStream.AUDIO)
|
||||
|
||||
await createOptimizeOrMergeAudioJobs({
|
||||
...pick(payload.optimizeJob, [ 'isNewVideo' ]),
|
||||
|
||||
video,
|
||||
videoFile,
|
||||
user
|
||||
})
|
||||
}
|
||||
|
||||
for (const job of (payload.jobs || [])) {
|
||||
await JobQueue.Instance.createJob(job)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode')
|
||||
}
|
||||
|
||||
for (const sequentialJobs of (payload.sequentialJobs || [])) {
|
||||
await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs)
|
||||
|
||||
await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processTranscodingJobBuilder
|
||||
}
|
||||
43
server/core/lib/job-queue/handlers/video-channel-import.ts
Normal file
43
server/core/lib/job-queue/handlers/video-channel-import.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { synchronizeChannel } from '@server/lib/sync-channel.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
|
||||
import { MChannelSync } from '@server/types/models/index.js'
|
||||
import { VideoChannelImportPayload } from '@peertube/peertube-models'
|
||||
|
||||
export async function processVideoChannelImport (job: Job) {
|
||||
const payload = job.data as VideoChannelImportPayload
|
||||
|
||||
logger.info('Processing video channel import in job %s.', job.id)
|
||||
|
||||
// Channel import requires only http upload to be allowed
|
||||
if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) {
|
||||
throw new Error('Cannot import channel as the HTTP upload is disabled')
|
||||
}
|
||||
|
||||
if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
|
||||
throw new Error('Cannot import channel as the synchronization is disabled')
|
||||
}
|
||||
|
||||
let channelSync: MChannelSync
|
||||
if (payload.partOfChannelSyncId) {
|
||||
channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId)
|
||||
|
||||
if (!channelSync) {
|
||||
throw new Error('Unknown channel sync specified in videos channel import')
|
||||
}
|
||||
}
|
||||
|
||||
const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId)
|
||||
|
||||
logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `)
|
||||
|
||||
await synchronizeChannel({
|
||||
channel: videoChannel,
|
||||
externalChannelUrl: payload.externalChannelUrl,
|
||||
channelSync,
|
||||
videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT
|
||||
})
|
||||
}
|
||||
78
server/core/lib/job-queue/handlers/video-file-import.ts
Normal file
78
server/core/lib/job-queue/handlers/video-file-import.ts
Normal file
@@ -0,0 +1,78 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { copy } from 'fs-extra/esm'
|
||||
import { VideoFileImportPayload } from '@peertube/peertube-models'
|
||||
import { createTorrentAndSetInfoHash } from '@server/lib/webtorrent.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
import { buildMoveVideoJob } from '@server/lib/video-jobs.js'
|
||||
import { buildNewFile } from '@server/lib/video-file.js'
|
||||
|
||||
async function processVideoFileImport (job: Job) {
|
||||
const payload = job.data as VideoFileImportPayload
|
||||
logger.info('Processing video file import in job %s.', job.id)
|
||||
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info(`Do not process job ${job.id}, video does not exist.`)
|
||||
return undefined
|
||||
}
|
||||
|
||||
await updateVideoFile(video, payload.filePath)
|
||||
|
||||
if (CONFIG.OBJECT_STORAGE.ENABLED) {
|
||||
await JobQueue.Instance.createJob(
|
||||
await buildMoveVideoJob({
|
||||
type: 'move-to-object-storage',
|
||||
video,
|
||||
moveVideoState: {
|
||||
isNewVideo: false,
|
||||
previousVideoState: video.state
|
||||
}
|
||||
})
|
||||
)
|
||||
} else {
|
||||
await federateVideoIfNeeded(video, false)
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoFileImport
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) {
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath)
|
||||
const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution)
|
||||
|
||||
if (currentVideoFile) {
|
||||
// Remove old file and old torrent
|
||||
await video.removeWebVideoFile(currentVideoFile)
|
||||
// Remove the old video file from the array
|
||||
video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile)
|
||||
|
||||
await currentVideoFile.destroy()
|
||||
}
|
||||
|
||||
const newVideoFile = await buildNewFile({ mode: 'web-video', path: inputFilePath })
|
||||
newVideoFile.videoId = video.id
|
||||
|
||||
const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile)
|
||||
await copy(inputFilePath, outputPath)
|
||||
|
||||
video.VideoFiles.push(newVideoFile)
|
||||
await createTorrentAndSetInfoHash(video, newVideoFile)
|
||||
|
||||
await newVideoFile.save()
|
||||
}
|
||||
372
server/core/lib/job-queue/handlers/video-import.ts
Normal file
372
server/core/lib/job-queue/handlers/video-import.ts
Normal file
@@ -0,0 +1,372 @@
|
||||
import { buildAspectRatio } from '@peertube/peertube-core-utils'
|
||||
import { ffprobePromise, getChaptersFromContainer, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
|
||||
import {
|
||||
ThumbnailType,
|
||||
ThumbnailType_Type,
|
||||
VideoImportPayload,
|
||||
VideoImportPreventExceptionResult,
|
||||
VideoImportState,
|
||||
VideoImportTorrentPayload,
|
||||
VideoImportTorrentPayloadType,
|
||||
VideoImportYoutubeDLPayload,
|
||||
VideoImportYoutubeDLPayloadType,
|
||||
VideoState
|
||||
} from '@peertube/peertube-models'
|
||||
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
|
||||
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
|
||||
import { setAndSaveVideoAutomaticTags } from '@server/lib/automatic-tags/automatic-tags.js'
|
||||
import { isPostImportVideoAccepted } from '@server/lib/moderation.js'
|
||||
import { Hooks } from '@server/lib/plugins/hooks.js'
|
||||
import { ServerConfigManager } from '@server/lib/server-config-manager.js'
|
||||
import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
|
||||
import { isUserQuotaValid } from '@server/lib/user.js'
|
||||
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
|
||||
import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js'
|
||||
import { buildNewFile } from '@server/lib/video-file.js'
|
||||
import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { buildNextVideoState } from '@server/lib/video-state.js'
|
||||
import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '@server/lib/webtorrent.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { move, remove } from 'fs-extra/esm'
|
||||
import { stat } from 'fs/promises'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { getSecureTorrentName } from '../../../helpers/utils.js'
|
||||
import { CONSTRAINTS_FIELDS, JOB_TTL } from '../../../initializers/constants.js'
|
||||
import { sequelizeTypescript } from '../../../initializers/database.js'
|
||||
import { VideoFileModel } from '../../../models/video/video-file.js'
|
||||
import { VideoImportModel } from '../../../models/video/video-import.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
import { federateVideoIfNeeded } from '../../activitypub/videos/index.js'
|
||||
import { Notifier } from '../../notifier/index.js'
|
||||
import { generateLocalVideoMiniature } from '../../thumbnail.js'
|
||||
import { JobQueue } from '../job-queue.js'
|
||||
|
||||
async function processVideoImport (job: Job): Promise<VideoImportPreventExceptionResult> {
|
||||
const payload = job.data as VideoImportPayload
|
||||
|
||||
const videoImport = await getVideoImportOrDie(payload)
|
||||
if (videoImport.state === VideoImportState.CANCELLED) {
|
||||
logger.info('Do not process import since it has been cancelled', { payload })
|
||||
return { resultType: 'success' }
|
||||
}
|
||||
|
||||
if (videoImport.attempts >= CONFIG.IMPORT.VIDEOS.MAX_ATTEMPTS) {
|
||||
logger.info('Do not process import since it has reached the maximum number of attempts', { payload, attempts: videoImport.attempts })
|
||||
|
||||
return { resultType: 'error' }
|
||||
}
|
||||
|
||||
videoImport.attempts += 1
|
||||
videoImport.state = VideoImportState.PROCESSING
|
||||
await videoImport.save()
|
||||
|
||||
try {
|
||||
if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload)
|
||||
if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload)
|
||||
|
||||
return { resultType: 'success' }
|
||||
} catch (err) {
|
||||
// Processors already handle video import state change on error
|
||||
|
||||
if (!payload.preventException) throw err
|
||||
|
||||
logger.warn('Catch error in video import to send value to parent job.', { payload, err })
|
||||
return { resultType: 'error' }
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoImport
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) {
|
||||
logger.info('Processing torrent video import in job %s.', job.id)
|
||||
|
||||
const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: payload.videoImportId }
|
||||
|
||||
const target = {
|
||||
torrentName: videoImport.torrentName
|
||||
? getSecureTorrentName(videoImport.torrentName)
|
||||
: undefined,
|
||||
uri: videoImport.magnetUri
|
||||
}
|
||||
return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)
|
||||
}
|
||||
|
||||
async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {
|
||||
logger.info('Processing youtubeDL video import in job %s.', job.id)
|
||||
|
||||
const options = { type: payload.type, generateTranscription: payload.generateTranscription, videoImportId: videoImport.id }
|
||||
|
||||
const youtubeDL = new YoutubeDLWrapper(
|
||||
videoImport.targetUrl,
|
||||
ServerConfigManager.Instance.getEnabledResolutions('vod'),
|
||||
CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION
|
||||
)
|
||||
|
||||
return processFile(
|
||||
() => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),
|
||||
videoImport,
|
||||
options
|
||||
)
|
||||
}
|
||||
|
||||
async function getVideoImportOrDie (payload: VideoImportPayload) {
|
||||
const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId)
|
||||
if (!videoImport) throw new Error('Video import not found')
|
||||
|
||||
if (!videoImport.Video) {
|
||||
const err = new Error(
|
||||
`Cannot process video import ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`
|
||||
)
|
||||
|
||||
await onImportError(err, null, videoImport)
|
||||
|
||||
throw err
|
||||
}
|
||||
|
||||
return videoImport
|
||||
}
|
||||
|
||||
type ProcessFileOptions = {
|
||||
type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType
|
||||
generateTranscription: boolean
|
||||
videoImportId: number
|
||||
}
|
||||
async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
|
||||
let tmpVideoPath: string
|
||||
let videoFile: MVideoFile
|
||||
|
||||
try {
|
||||
// Download video from youtubeDL
|
||||
tmpVideoPath = await downloader()
|
||||
|
||||
// Get information about this video
|
||||
const stats = await stat(tmpVideoPath)
|
||||
const isAble = await isUserQuotaValid({ userId: videoImport.User.id, uploadSize: stats.size })
|
||||
if (isAble === false) {
|
||||
throw new Error('The user video quota is exceeded with this video to import.')
|
||||
}
|
||||
|
||||
const ffprobe = await ffprobePromise(tmpVideoPath)
|
||||
const duration = await getVideoStreamDuration(tmpVideoPath, ffprobe)
|
||||
|
||||
const containerChapters = await getChaptersFromContainer({
|
||||
path: tmpVideoPath,
|
||||
maxTitleLength: CONSTRAINTS_FIELDS.VIDEO_CHAPTERS.TITLE.max,
|
||||
ffprobe
|
||||
})
|
||||
|
||||
videoFile = await buildNewFile({ mode: 'web-video', ffprobe, path: tmpVideoPath })
|
||||
videoFile.videoId = videoImport.videoId
|
||||
|
||||
const hookName = options.type === 'youtube-dl'
|
||||
? 'filter:api.video.post-import-url.accept.result'
|
||||
: 'filter:api.video.post-import-torrent.accept.result'
|
||||
|
||||
// Check we accept this video
|
||||
const acceptParameters = {
|
||||
videoImport,
|
||||
video: videoImport.Video,
|
||||
videoFilePath: tmpVideoPath,
|
||||
videoFile: videoFile as VideoFileModel,
|
||||
user: videoImport.User
|
||||
}
|
||||
const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)
|
||||
|
||||
if (acceptedResult.accepted !== true) {
|
||||
logger.info('Refused imported video.', { acceptedResult, acceptParameters })
|
||||
|
||||
videoImport.state = VideoImportState.REJECTED
|
||||
await videoImport.save()
|
||||
|
||||
throw new Error(acceptedResult.errorMessage)
|
||||
}
|
||||
|
||||
// Video is accepted, resuming preparation
|
||||
const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid)
|
||||
|
||||
try {
|
||||
const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile)
|
||||
|
||||
// Move file
|
||||
const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile)
|
||||
await move(tmpVideoPath, videoDestFile)
|
||||
|
||||
tmpVideoPath = null // This path is not used anymore
|
||||
|
||||
const thumbnails = await generateMiniature({ videoImportWithFiles, videoFile, ffprobe })
|
||||
|
||||
// Create torrent
|
||||
await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile)
|
||||
|
||||
const { videoImportUpdated, video } = await retryTransactionWrapper(() => {
|
||||
return sequelizeTypescript.transaction(async t => {
|
||||
// Refresh video
|
||||
const video = await VideoModel.load(videoImportWithFiles.videoId, t)
|
||||
if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.')
|
||||
|
||||
await videoFile.save({ transaction: t })
|
||||
|
||||
// Update video DB object
|
||||
video.duration = duration
|
||||
video.state = buildNextVideoState(video.state)
|
||||
video.aspectRatio = buildAspectRatio({ width: videoFile.width, height: videoFile.height })
|
||||
await video.save({ transaction: t })
|
||||
|
||||
for (const thumbnail of thumbnails) {
|
||||
await video.addAndSaveThumbnail(thumbnail, t)
|
||||
}
|
||||
|
||||
await replaceChaptersIfNotExist({ video, chapters: containerChapters, transaction: t })
|
||||
|
||||
const automaticTags = await new AutomaticTagger().buildVideoAutomaticTags({ video, transaction: t })
|
||||
await setAndSaveVideoAutomaticTags({ video, automaticTags, transaction: t })
|
||||
|
||||
// Now we can federate the video (reload from database, we need more attributes)
|
||||
const videoForFederation = await VideoModel.loadFull(video.uuid, t)
|
||||
await federateVideoIfNeeded(videoForFederation, true, t)
|
||||
|
||||
// Update video import object
|
||||
videoImportWithFiles.state = VideoImportState.SUCCESS
|
||||
const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport
|
||||
|
||||
logger.info('Video %s imported.', video.uuid)
|
||||
|
||||
return { videoImportUpdated, video: videoForFederation }
|
||||
})
|
||||
})
|
||||
|
||||
await afterImportSuccess({
|
||||
videoImport: videoImportUpdated,
|
||||
video,
|
||||
videoFile,
|
||||
user: videoImport.User,
|
||||
generateTranscription: options.generateTranscription
|
||||
})
|
||||
} finally {
|
||||
videoFileLockReleaser()
|
||||
}
|
||||
} catch (err) {
|
||||
await onImportError(err, tmpVideoPath, videoImport)
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise<MVideoImportDefaultFiles> {
|
||||
// Refresh video, privacy may have changed
|
||||
const video = await videoImport.Video.reload()
|
||||
const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] })
|
||||
|
||||
return Object.assign(videoImport, { Video: videoWithFiles })
|
||||
}
|
||||
|
||||
async function generateMiniature (options: {
|
||||
videoImportWithFiles: MVideoImportDefaultFiles
|
||||
videoFile: MVideoFile
|
||||
ffprobe: FfprobeData
|
||||
}) {
|
||||
const { ffprobe, videoFile, videoImportWithFiles } = options
|
||||
|
||||
const thumbnailsToGenerate: ThumbnailType_Type[] = []
|
||||
|
||||
if (!videoImportWithFiles.Video.getMiniature()) {
|
||||
thumbnailsToGenerate.push(ThumbnailType.MINIATURE)
|
||||
}
|
||||
|
||||
if (!videoImportWithFiles.Video.getPreview()) {
|
||||
thumbnailsToGenerate.push(ThumbnailType.PREVIEW)
|
||||
}
|
||||
|
||||
return generateLocalVideoMiniature({
|
||||
video: videoImportWithFiles.Video,
|
||||
videoFile,
|
||||
types: thumbnailsToGenerate,
|
||||
ffprobe
|
||||
})
|
||||
}
|
||||
|
||||
async function afterImportSuccess (options: {
|
||||
videoImport: MVideoImport
|
||||
video: MVideoFullLight
|
||||
videoFile: MVideoFile
|
||||
user: MUserId
|
||||
|
||||
generateTranscription: boolean
|
||||
}) {
|
||||
const { video, videoFile, videoImport, user, generateTranscription } = options
|
||||
|
||||
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true })
|
||||
|
||||
if (video.isBlacklisted()) {
|
||||
const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video })
|
||||
|
||||
Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist)
|
||||
} else {
|
||||
Notifier.Instance.notifyOnNewVideoOrLiveIfNeeded(video)
|
||||
}
|
||||
|
||||
// Generate the storyboard in the job queue, and don't forget to federate an update after
|
||||
await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
|
||||
|
||||
if (await VideoCaptionModel.hasVideoCaption(video.id) !== true && generateTranscription === true) {
|
||||
await createTranscriptionTaskIfNeeded(video)
|
||||
}
|
||||
|
||||
if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
|
||||
await JobQueue.Instance.createJob(
|
||||
await buildMoveVideoJob({
|
||||
type: 'move-to-object-storage',
|
||||
video,
|
||||
moveVideoState: {
|
||||
isNewVideo: true,
|
||||
previousVideoState: VideoState.TO_IMPORT
|
||||
}
|
||||
})
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs?
|
||||
await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user })
|
||||
}
|
||||
}
|
||||
|
||||
async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) {
|
||||
try {
|
||||
if (tempVideoPath) await remove(tempVideoPath)
|
||||
} catch (errUnlink) {
|
||||
logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink })
|
||||
}
|
||||
|
||||
await sequelizeTypescript.transaction(async t => {
|
||||
videoImport.error = err.message
|
||||
|
||||
if (videoImport.state !== VideoImportState.REJECTED) {
|
||||
videoImport.state = VideoImportState.FAILED
|
||||
}
|
||||
|
||||
await videoImport.save({ transaction: t })
|
||||
|
||||
const video = await VideoModel.load(videoImport.videoId, t)
|
||||
|
||||
if (video) {
|
||||
video.state = VideoState.TO_IMPORT_FAILED
|
||||
await video.save({ transaction: t })
|
||||
}
|
||||
})
|
||||
|
||||
Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false })
|
||||
}
|
||||
369
server/core/lib/job-queue/handlers/video-live-ending.ts
Normal file
369
server/core/lib/job-queue/handlers/video-live-ending.ts
Normal file
@@ -0,0 +1,369 @@
|
||||
import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
|
||||
import { ThumbnailType, VideoFileStream, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models'
|
||||
import { peertubeTruncate } from '@server/helpers/core-utils.js'
|
||||
import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
|
||||
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
|
||||
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
|
||||
import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live/index.js'
|
||||
import {
|
||||
generateHLSMasterPlaylistFilename,
|
||||
generateHlsSha256SegmentsFilename,
|
||||
getHLSDirectory,
|
||||
getLiveReplayBaseDirectory
|
||||
} from '@server/lib/paths.js'
|
||||
import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
|
||||
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'
|
||||
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
|
||||
import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
|
||||
import { moveToNextState } from '@server/lib/video-state.js'
|
||||
import { setVideoTags } from '@server/lib/video.js'
|
||||
import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js'
|
||||
import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js'
|
||||
import { VideoLiveModel } from '@server/models/video/video-live.js'
|
||||
import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import {
|
||||
MThumbnail,
|
||||
MVideo,
|
||||
MVideoLive,
|
||||
MVideoLiveSession,
|
||||
MVideoTag,
|
||||
MVideoThumbnail,
|
||||
MVideoWithAllFiles,
|
||||
MVideoWithFileThumbnail
|
||||
} from '@server/types/models/index.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { pathExists, remove } from 'fs-extra/esm'
|
||||
import { readdir } from 'fs/promises'
|
||||
import { isAbsolute, join } from 'path'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
|
||||
const lTags = loggerTagsFactory('live', 'job')
|
||||
|
||||
export async function processVideoLiveEnding (job: Job) {
|
||||
const payload = job.data as VideoLiveEndingPayload
|
||||
|
||||
logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() })
|
||||
|
||||
function logError () {
|
||||
logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags())
|
||||
}
|
||||
|
||||
const video = await VideoModel.load(payload.videoId)
|
||||
const live = await VideoLiveModel.loadByVideoId(payload.videoId)
|
||||
const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId)
|
||||
|
||||
if (!video || !live || !liveSession) {
|
||||
logError()
|
||||
return
|
||||
}
|
||||
|
||||
const permanentLive = live.permanentLive
|
||||
|
||||
liveSession.endingProcessed = true
|
||||
await liveSession.save()
|
||||
|
||||
if (liveSession.saveReplay !== true) {
|
||||
return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
|
||||
}
|
||||
|
||||
let replayDirectory = payload.replayDirectory
|
||||
|
||||
// Introduced in PeerTube 7.2, allow to use the appropriate base directory even if the live privacy changed
|
||||
if (!isAbsolute(replayDirectory)) {
|
||||
replayDirectory = join(getLiveReplayBaseDirectory(video), replayDirectory)
|
||||
}
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid)
|
||||
|
||||
try {
|
||||
await video.reload()
|
||||
|
||||
if (await hasReplayFiles(replayDirectory) !== true) {
|
||||
logger.info(`No replay files found for live ${video.uuid}, skipping video replay creation.`, { ...lTags(video.uuid) })
|
||||
|
||||
await cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
|
||||
} else if (permanentLive) {
|
||||
await saveReplayToExternalVideo({
|
||||
liveVideo: video,
|
||||
liveSession,
|
||||
publishedAt: payload.publishedAt,
|
||||
replayDirectory
|
||||
})
|
||||
|
||||
await cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId })
|
||||
} else {
|
||||
await replaceLiveByReplay({
|
||||
video,
|
||||
liveSession,
|
||||
live,
|
||||
permanentLive,
|
||||
replayDirectory
|
||||
})
|
||||
}
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function saveReplayToExternalVideo (options: {
|
||||
liveVideo: MVideoThumbnail
|
||||
liveSession: MVideoLiveSession
|
||||
publishedAt: string
|
||||
replayDirectory: string
|
||||
}) {
|
||||
const { liveSession, publishedAt, replayDirectory } = options
|
||||
|
||||
const liveVideo = await VideoModel.loadFull(options.liveVideo.id)
|
||||
const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId)
|
||||
|
||||
const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}`
|
||||
const truncatedVideoName = peertubeTruncate(liveVideo.name, {
|
||||
length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length
|
||||
})
|
||||
|
||||
const replayVideo = new VideoModel({
|
||||
name: truncatedVideoName + videoNameSuffix,
|
||||
isLive: false,
|
||||
state: VideoState.TO_TRANSCODE,
|
||||
duration: 0,
|
||||
|
||||
remote: liveVideo.remote,
|
||||
category: liveVideo.category,
|
||||
licence: liveVideo.licence,
|
||||
language: liveVideo.language,
|
||||
commentsPolicy: liveVideo.commentsPolicy,
|
||||
downloadEnabled: liveVideo.downloadEnabled,
|
||||
waitTranscoding: true,
|
||||
nsfw: liveVideo.nsfw,
|
||||
description: liveVideo.description,
|
||||
aspectRatio: liveVideo.aspectRatio,
|
||||
support: liveVideo.support,
|
||||
privacy: replaySettings.privacy,
|
||||
channelId: liveVideo.channelId
|
||||
}) as MVideoWithAllFiles & MVideoTag
|
||||
|
||||
replayVideo.Thumbnails = []
|
||||
replayVideo.VideoFiles = []
|
||||
replayVideo.VideoStreamingPlaylists = []
|
||||
|
||||
replayVideo.url = getLocalVideoActivityPubUrl(replayVideo)
|
||||
|
||||
await replayVideo.save()
|
||||
|
||||
await setVideoTags({ video: replayVideo, tags: liveVideo.Tags.map(t => t.name) })
|
||||
|
||||
liveSession.replayVideoId = replayVideo.id
|
||||
await liveSession.save()
|
||||
|
||||
// If live is blacklisted, also blacklist the replay
|
||||
const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id)
|
||||
if (blacklist) {
|
||||
await VideoBlacklistModel.create({
|
||||
videoId: replayVideo.id,
|
||||
unfederated: blacklist.unfederated,
|
||||
reason: blacklist.reason,
|
||||
type: blacklist.type
|
||||
})
|
||||
}
|
||||
|
||||
await assignReplayFilesToVideo({ video: replayVideo, replayDirectory })
|
||||
|
||||
logger.info(`Removing replay directory ${replayDirectory}`, lTags(liveVideo.uuid))
|
||||
await remove(replayDirectory)
|
||||
|
||||
try {
|
||||
await copyOrRegenerateThumbnails({ liveVideo, replayVideo })
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`Cannot copy/regenerate thumbnails of ended live ${liveVideo.uuid} to external video ${replayVideo.uuid}`,
|
||||
lTags(liveVideo.uuid, replayVideo.uuid)
|
||||
)
|
||||
}
|
||||
|
||||
await createStoryboardJob(replayVideo)
|
||||
await createTranscriptionTaskIfNeeded(replayVideo)
|
||||
|
||||
await moveToNextState({ video: replayVideo, isNewVideo: true })
|
||||
}
|
||||
|
||||
async function copyOrRegenerateThumbnails (options: {
|
||||
liveVideo: MVideoThumbnail
|
||||
replayVideo: MVideoWithFileThumbnail
|
||||
}) {
|
||||
const { liveVideo, replayVideo } = options
|
||||
|
||||
let thumbnails: MThumbnail[] = []
|
||||
const preview = liveVideo.getPreview()
|
||||
|
||||
if (preview?.automaticallyGenerated === false) {
|
||||
thumbnails = await Promise.all(
|
||||
[ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ].map(type => {
|
||||
return updateLocalVideoMiniatureFromExisting({
|
||||
inputPath: preview.getPath(),
|
||||
video: replayVideo,
|
||||
type,
|
||||
automaticallyGenerated: false,
|
||||
keepOriginal: true
|
||||
})
|
||||
})
|
||||
)
|
||||
} else {
|
||||
thumbnails = await generateLocalVideoMiniature({
|
||||
video: replayVideo,
|
||||
videoFile: replayVideo.getMaxQualityFile(VideoFileStream.VIDEO) || replayVideo.getMaxQualityFile(VideoFileStream.AUDIO),
|
||||
types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ],
|
||||
ffprobe: undefined
|
||||
})
|
||||
}
|
||||
|
||||
for (const thumbnail of thumbnails) {
|
||||
await replayVideo.addAndSaveThumbnail(thumbnail)
|
||||
}
|
||||
}
|
||||
|
||||
async function replaceLiveByReplay (options: {
|
||||
video: MVideo
|
||||
liveSession: MVideoLiveSession
|
||||
live: MVideoLive
|
||||
permanentLive: boolean
|
||||
replayDirectory: string
|
||||
}) {
|
||||
const { video: liveVideo, liveSession, live, permanentLive, replayDirectory } = options
|
||||
|
||||
const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId)
|
||||
const videoWithFiles = await VideoModel.loadFull(liveVideo.id)
|
||||
const hlsPlaylist = videoWithFiles.getHLSPlaylist()
|
||||
const replayInAnotherDirectory = isVideoInPublicDirectory(liveVideo.privacy) !== isVideoInPublicDirectory(replaySettings.privacy)
|
||||
|
||||
logger.info(`Replacing live ${liveVideo.uuid} by replay ${replayDirectory}.`, { replayInAnotherDirectory, ...lTags(liveVideo.uuid) })
|
||||
|
||||
await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist)
|
||||
|
||||
await live.destroy()
|
||||
|
||||
videoWithFiles.isLive = false
|
||||
videoWithFiles.privacy = replaySettings.privacy
|
||||
videoWithFiles.waitTranscoding = true
|
||||
videoWithFiles.state = VideoState.TO_TRANSCODE
|
||||
|
||||
await videoWithFiles.save()
|
||||
|
||||
liveSession.replayVideoId = videoWithFiles.id
|
||||
await liveSession.save()
|
||||
|
||||
await VideoFileModel.removeHLSFilesOfStreamingPlaylistId(hlsPlaylist.id)
|
||||
|
||||
// Reset playlist
|
||||
hlsPlaylist.VideoFiles = []
|
||||
hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename()
|
||||
hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename()
|
||||
await hlsPlaylist.save()
|
||||
|
||||
await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory })
|
||||
|
||||
// Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay
|
||||
if (permanentLive) { // Remove session replay
|
||||
await remove(replayDirectory)
|
||||
} else {
|
||||
// We won't stream again in this live, we can delete the base replay directory
|
||||
await remove(getLiveReplayBaseDirectory(liveVideo))
|
||||
|
||||
// If the live was in another base directory, also delete it
|
||||
if (replayInAnotherDirectory) {
|
||||
await remove(getHLSDirectory(liveVideo))
|
||||
}
|
||||
}
|
||||
|
||||
// Regenerate the thumbnail & preview?
|
||||
try {
|
||||
await regenerateMiniaturesIfNeeded(videoWithFiles, undefined)
|
||||
} catch (err) {
|
||||
logger.error(`Cannot regenerate thumbnails of ended live ${videoWithFiles.uuid}`, lTags(liveVideo.uuid))
|
||||
}
|
||||
|
||||
// We consider this is a new video
|
||||
await moveToNextState({ video: videoWithFiles, isNewVideo: true })
|
||||
|
||||
await createStoryboardJob(videoWithFiles)
|
||||
await createTranscriptionTaskIfNeeded(videoWithFiles)
|
||||
}
|
||||
|
||||
async function assignReplayFilesToVideo (options: {
|
||||
video: MVideo
|
||||
replayDirectory: string
|
||||
}) {
|
||||
const { video, replayDirectory } = options
|
||||
|
||||
const concatenatedTsFiles = await readdir(replayDirectory)
|
||||
|
||||
logger.info(`Assigning replays ${replayDirectory} to video ${video.uuid}.`, { concatenatedTsFiles, ...lTags(video.uuid) })
|
||||
|
||||
for (const concatenatedTsFile of concatenatedTsFiles) {
|
||||
// Generating hls playlist can be long, reload the video in this case
|
||||
await video.reload()
|
||||
|
||||
const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile)
|
||||
|
||||
const probe = await ffprobePromise(concatenatedTsFilePath)
|
||||
const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe)
|
||||
const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe)
|
||||
const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe)
|
||||
|
||||
try {
|
||||
await generateHlsPlaylistResolutionFromTS({
|
||||
video,
|
||||
inputFileMutexReleaser: null, // Already locked in parent
|
||||
concatenatedTsFilePath,
|
||||
resolution,
|
||||
fps,
|
||||
isAAC: audioStream?.codec_name === 'aac'
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error('Cannot generate HLS playlist resolution from TS files.', { err })
|
||||
}
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
async function cleanupLiveAndFederate (options: {
|
||||
video: MVideo
|
||||
permanentLive: boolean
|
||||
streamingPlaylistId: number
|
||||
}) {
|
||||
const { permanentLive, video, streamingPlaylistId } = options
|
||||
|
||||
const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId)
|
||||
|
||||
if (streamingPlaylist) {
|
||||
if (permanentLive) {
|
||||
await cleanupAndDestroyPermanentLive(video, streamingPlaylist)
|
||||
} else {
|
||||
await cleanupUnsavedNormalLive(video, streamingPlaylist)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const fullVideo = await VideoModel.loadFull(video.id)
|
||||
return federateVideoIfNeeded(fullVideo, false, undefined)
|
||||
} catch (err) {
|
||||
logger.warn('Cannot federate live after cleanup', { videoId: video.id, err })
|
||||
}
|
||||
}
|
||||
|
||||
function createStoryboardJob (video: MVideo) {
|
||||
return addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
|
||||
}
|
||||
|
||||
async function hasReplayFiles (replayDirectory: string) {
|
||||
return await pathExists(replayDirectory) && (await readdir(replayDirectory)).length !== 0
|
||||
}
|
||||
17
server/core/lib/job-queue/handlers/video-redundancy.ts
Normal file
17
server/core/lib/job-queue/handlers/video-redundancy.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import { Job } from 'bullmq'
|
||||
import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler.js'
|
||||
import { VideoRedundancyPayload } from '@peertube/peertube-models'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
|
||||
async function processVideoRedundancy (job: Job) {
|
||||
const payload = job.data as VideoRedundancyPayload
|
||||
logger.info('Processing video redundancy in job %s.', job.id)
|
||||
|
||||
return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoRedundancy
|
||||
}
|
||||
208
server/core/lib/job-queue/handlers/video-studio-edition.ts
Normal file
208
server/core/lib/job-queue/handlers/video-studio-edition.ts
Normal file
@@ -0,0 +1,208 @@
|
||||
import { pick } from '@peertube/peertube-core-utils'
|
||||
import { FFmpegEdition } from '@peertube/peertube-ffmpeg'
|
||||
import {
|
||||
VideoState,
|
||||
VideoStudioEditionPayload,
|
||||
VideoStudioTask,
|
||||
VideoStudioTaskCutPayload,
|
||||
VideoStudioTaskIntroPayload,
|
||||
VideoStudioTaskOutroPayload,
|
||||
VideoStudioTaskPayload,
|
||||
VideoStudioTaskWatermarkPayload
|
||||
} from '@peertube/peertube-models'
|
||||
import { buildUUID } from '@peertube/peertube-node-utils'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
|
||||
import { isUserQuotaValid } from '@server/lib/user.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { VideoModel } from '@server/models/video/video.js'
|
||||
import { MVideo, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { MutexInterface } from 'async-mutex'
|
||||
import { Job } from 'bullmq'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { extname, join } from 'path'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
|
||||
const lTagsBase = loggerTagsFactory('video-studio')
|
||||
|
||||
async function processVideoStudioEdition (job: Job) {
|
||||
const payload = job.data as VideoStudioEditionPayload
|
||||
const lTags = lTagsBase(payload.videoUUID)
|
||||
|
||||
logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
|
||||
|
||||
let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info('Can\'t process job %d, video does not exist.', job.id, lTags)
|
||||
|
||||
await safeCleanupStudioTMPFiles(payload.tasks)
|
||||
return undefined
|
||||
}
|
||||
|
||||
await checkUserQuotaOrThrow(video, payload)
|
||||
|
||||
await video.reload()
|
||||
|
||||
const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({
|
||||
videoPath: originalVideoFilePath,
|
||||
separatedAudioPath
|
||||
}) => {
|
||||
let tmpInputFilePath: string
|
||||
let outputPath: string
|
||||
|
||||
for (const task of payload.tasks) {
|
||||
const outputFilename = buildUUID() + extname(originalVideoFilePath)
|
||||
outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
|
||||
|
||||
await processTask({
|
||||
videoInputPath: tmpInputFilePath ?? originalVideoFilePath,
|
||||
|
||||
separatedAudioInputPath: tmpInputFilePath
|
||||
? undefined
|
||||
: separatedAudioPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
|
||||
video,
|
||||
outputPath,
|
||||
task,
|
||||
lTags
|
||||
})
|
||||
|
||||
if (tmpInputFilePath) await remove(tmpInputFilePath)
|
||||
|
||||
// For the next iteration
|
||||
tmpInputFilePath = outputPath
|
||||
inputFileMutexReleaser = undefined
|
||||
}
|
||||
|
||||
return outputPath
|
||||
})
|
||||
|
||||
logger.info('Video edition ended for video %s.', video.uuid, lTags)
|
||||
|
||||
await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks })
|
||||
} catch (err) {
|
||||
await safeCleanupStudioTMPFiles(payload.tasks)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
await video.setNewState(VideoState.PUBLISHED, false, undefined)
|
||||
} catch (err) {
|
||||
logger.error('Cannot reset video state after studio error', { err, ...lTags })
|
||||
}
|
||||
|
||||
throw err
|
||||
} finally {
|
||||
if (inputFileMutexReleaser) inputFileMutexReleaser()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoStudioEdition
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type TaskProcessorOptions<T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = {
|
||||
videoInputPath: string
|
||||
separatedAudioInputPath?: string
|
||||
|
||||
inputFileMutexReleaser: MutexInterface.Releaser
|
||||
|
||||
outputPath: string
|
||||
video: MVideo
|
||||
task: T
|
||||
lTags: { tags: (string | number)[] }
|
||||
}
|
||||
|
||||
const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise<any> } = {
|
||||
'add-intro': processAddIntroOutro,
|
||||
'add-outro': processAddIntroOutro,
|
||||
'cut': processCut,
|
||||
'add-watermark': processAddWatermark
|
||||
}
|
||||
|
||||
async function processTask (options: TaskProcessorOptions) {
|
||||
const { video, task, lTags } = options
|
||||
|
||||
logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags })
|
||||
|
||||
const processor = taskProcessors[options.task.name]
|
||||
if (!process) throw new Error('Unknown task ' + task.name)
|
||||
|
||||
return processor(options)
|
||||
}
|
||||
|
||||
function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
|
||||
const { task, lTags } = options
|
||||
|
||||
logger.debug('Will add intro/outro to the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().addIntroOutro({
|
||||
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
|
||||
|
||||
introOutroPath: task.options.file,
|
||||
type: task.name === 'add-intro'
|
||||
? 'intro'
|
||||
: 'outro'
|
||||
})
|
||||
}
|
||||
|
||||
function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
|
||||
const { task, lTags } = options
|
||||
|
||||
logger.debug('Will cut the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().cutVideo({
|
||||
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
|
||||
|
||||
start: task.options.start,
|
||||
end: task.options.end
|
||||
})
|
||||
}
|
||||
|
||||
function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
|
||||
const { task, lTags } = options
|
||||
|
||||
logger.debug('Will add watermark to the video.', { options, ...lTags })
|
||||
|
||||
return buildFFmpegEdition().addWatermark({
|
||||
...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
|
||||
|
||||
watermarkPath: task.options.file,
|
||||
|
||||
videoFilters: {
|
||||
watermarkSizeRatio: task.options.watermarkSizeRatio,
|
||||
horitonzalMarginRatio: task.options.horitonzalMarginRatio,
|
||||
verticalMarginRatio: task.options.verticalMarginRatio
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) {
|
||||
const user = await UserModel.loadByVideoId(video.id)
|
||||
|
||||
const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file
|
||||
|
||||
const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder)
|
||||
if (await isUserQuotaValid({ userId: user.id, uploadSize: additionalBytes }) === false) {
|
||||
throw new Error('Quota exceeded for this user to edit the video')
|
||||
}
|
||||
}
|
||||
|
||||
function buildFFmpegEdition () {
|
||||
return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()))
|
||||
}
|
||||
165
server/core/lib/job-queue/handlers/video-transcoding.ts
Normal file
165
server/core/lib/job-queue/handlers/video-transcoding.ts
Normal file
@@ -0,0 +1,165 @@
|
||||
import {
|
||||
HLSTranscodingPayload,
|
||||
MergeAudioTranscodingPayload,
|
||||
NewWebVideoResolutionTranscodingPayload,
|
||||
OptimizeTranscodingPayload,
|
||||
VideoTranscodingPayload
|
||||
} from '@peertube/peertube-models'
|
||||
import { hasMissingHLSStreams } from '@server/lib/runners/job-handlers/shared/utils.js'
|
||||
import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js'
|
||||
import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding.js'
|
||||
import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding.js'
|
||||
import { removeAllWebVideoFiles } from '@server/lib/video-file.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { moveToFailedTranscodingState } from '@server/lib/video-state.js'
|
||||
import { UserModel } from '@server/models/user/user.js'
|
||||
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
|
||||
import { MUser, MUserId, MVideoFullLight } from '@server/types/models/index.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
|
||||
type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<void>
|
||||
|
||||
const handlers: { [id in VideoTranscodingPayload['type']]: HandlerFunction } = {
|
||||
'new-resolution-to-hls': handleHLSJob,
|
||||
'new-resolution-to-web-video': handleNewWebVideoResolutionJob,
|
||||
'merge-audio-to-web-video': handleWebVideoMergeAudioJob,
|
||||
'optimize-to-web-video': handleWebVideoOptimizeJob
|
||||
}
|
||||
|
||||
const lTags = loggerTagsFactory('transcoding')
|
||||
|
||||
async function processVideoTranscoding (job: Job) {
|
||||
const payload = job.data as VideoTranscodingPayload
|
||||
logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID))
|
||||
|
||||
const video = await VideoModel.loadFull(payload.videoUUID)
|
||||
// No video, maybe deleted?
|
||||
if (!video) {
|
||||
logger.info(`Do not process job ${job.id}, video does not exist.`, lTags(payload.videoUUID))
|
||||
return undefined
|
||||
}
|
||||
|
||||
const user = await UserModel.loadByChannelActorId(video.VideoChannel.Actor.id)
|
||||
|
||||
const handler = handlers[payload.type]
|
||||
|
||||
if (!handler) {
|
||||
await moveToFailedTranscodingState(video)
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
throw new Error('Cannot find transcoding handler for ' + payload.type)
|
||||
}
|
||||
|
||||
try {
|
||||
await handler(job, payload, video, user)
|
||||
} catch (error) {
|
||||
await moveToFailedTranscodingState(video)
|
||||
|
||||
await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode')
|
||||
|
||||
throw error
|
||||
}
|
||||
|
||||
return video
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideoTranscoding
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Job handlers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job })
|
||||
|
||||
logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
|
||||
}
|
||||
|
||||
async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
|
||||
logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await optimizeOriginalVideofile({ video, job })
|
||||
|
||||
logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) {
|
||||
logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job })
|
||||
|
||||
logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
// Always move video to next state, we're ready enough with this resolution
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: payload.canMoveVideoState, video })
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) {
|
||||
logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload })
|
||||
|
||||
const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid)
|
||||
let video: MVideoFullLight
|
||||
|
||||
try {
|
||||
video = await VideoModel.loadFull(videoArg.uuid)
|
||||
|
||||
const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
|
||||
const webVideoFile = video.getWebVideoFileResolution(payload.resolution)
|
||||
|
||||
const videoFileInputs = webVideoFile
|
||||
? [ webVideoFile ]
|
||||
: [ videoFile, separatedAudioFile ].filter(v => !!v)
|
||||
|
||||
await VideoPathManager.Instance.makeAvailableVideoFiles(videoFileInputs, ([ videoPath, separatedAudioPath ]) => {
|
||||
return generateHlsPlaylistResolution({
|
||||
video,
|
||||
|
||||
videoInputPath: videoPath,
|
||||
separatedAudioInputPath: separatedAudioPath,
|
||||
|
||||
inputFileMutexReleaser,
|
||||
resolution: payload.resolution,
|
||||
fps: payload.fps,
|
||||
separatedAudio: payload.separatedAudio,
|
||||
job
|
||||
})
|
||||
})
|
||||
} finally {
|
||||
inputFileMutexReleaser()
|
||||
}
|
||||
|
||||
logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
|
||||
|
||||
const missingStream = await hasMissingHLSStreams({
|
||||
inputStreams: payload.inputStreams,
|
||||
transcodingRequestAt: payload.transcodingRequestAt,
|
||||
videoId: videoArg.uuid
|
||||
})
|
||||
|
||||
if (!missingStream && payload.deleteWebVideoFiles === true) {
|
||||
logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid))
|
||||
|
||||
await removeAllWebVideoFiles(video)
|
||||
}
|
||||
|
||||
// Splitted audio, wait audio & video generation before moving the video in its next state
|
||||
const moveVideoToNextState = payload.canMoveVideoState && !missingStream
|
||||
|
||||
await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState, video })
|
||||
}
|
||||
21
server/core/lib/job-queue/handlers/video-transcription.ts
Normal file
21
server/core/lib/job-queue/handlers/video-transcription.ts
Normal file
@@ -0,0 +1,21 @@
|
||||
import { VideoTranscriptionPayload } from '@peertube/peertube-models'
|
||||
import { generateSubtitle } from '@server/lib/video-captions.js'
|
||||
import { Job } from 'bullmq'
|
||||
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
|
||||
const lTags = loggerTagsFactory('transcription')
|
||||
|
||||
export async function processVideoTranscription (job: Job) {
|
||||
const payload = job.data as VideoTranscriptionPayload
|
||||
|
||||
logger.info('Processing video transcription in job %s.', job.id)
|
||||
|
||||
const video = await VideoModel.load(payload.videoUUID)
|
||||
if (!video) {
|
||||
logger.info('Do not process transcription job %d, video does not exist.', job.id, lTags(payload.videoUUID))
|
||||
return
|
||||
}
|
||||
|
||||
return generateSubtitle({ video })
|
||||
}
|
||||
57
server/core/lib/job-queue/handlers/video-views-stats.ts
Normal file
57
server/core/lib/job-queue/handlers/video-views-stats.ts
Normal file
@@ -0,0 +1,57 @@
|
||||
import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
|
||||
import { VideoViewModel } from '@server/models/view/video-view.js'
|
||||
import { logger } from '../../../helpers/logger.js'
|
||||
import { VideoModel } from '../../../models/video/video.js'
|
||||
import { Redis } from '../../redis.js'
|
||||
|
||||
async function processVideosViewsStats () {
|
||||
const lastHour = new Date()
|
||||
|
||||
// In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour
|
||||
if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1)
|
||||
|
||||
const hour = lastHour.getHours()
|
||||
const startDate = lastHour.setMinutes(0, 0, 0)
|
||||
const endDate = lastHour.setMinutes(59, 59, 999)
|
||||
|
||||
const videoIds = await Redis.Instance.listVideosViewedForStats(hour)
|
||||
if (videoIds.length === 0) return
|
||||
|
||||
logger.info('Processing videos views stats in job for hour %d.', hour)
|
||||
|
||||
for (const videoId of videoIds) {
|
||||
try {
|
||||
const views = await Redis.Instance.getVideoViewsStats(videoId, hour)
|
||||
await Redis.Instance.deleteVideoViewsStats(videoId, hour)
|
||||
|
||||
if (views) {
|
||||
logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour)
|
||||
|
||||
try {
|
||||
const video = await VideoModel.load(videoId)
|
||||
if (!video) {
|
||||
logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId)
|
||||
continue
|
||||
}
|
||||
|
||||
await VideoViewModel.create({
|
||||
startDate: new Date(startDate),
|
||||
endDate: new Date(endDate),
|
||||
views,
|
||||
videoId
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err })
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
processVideosViewsStats
|
||||
}
|
||||
Reference in New Issue
Block a user