Init commit

server/core/lib/schedulers/abstract-scheduler.ts

@@ -0,0 +1,71 @@
+import Bluebird from 'bluebird'
+import { randomInt } from 'node:crypto'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export abstract class AbstractScheduler {
+  protected abstract schedulerIntervalMs: number
+
+  private interval: NodeJS.Timeout
+  private firstRunTimeout: NodeJS.Timeout
+  private isRunning = false
+
+  private readonly randomRunOnEnable: boolean
+
+  constructor (options: {
+    randomRunOnEnable?: boolean
+  }) {
+    if (options?.randomRunOnEnable === true) {
+      this.randomRunOnEnable = true
+    }
+  }
+
+  enable () {
+    if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.')
+
+    if (this.randomRunOnEnable === true) {
+      const randomDelay = randomInt(0, Math.floor(this.schedulerIntervalMs / 2))
+
+      this.firstRunTimeout = setTimeout(async () => {
+        try {
+          // execute() already handles errors
+          await this.execute()
+        } finally {
+          this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs)
+        }
+      }, randomDelay)
+
+      return
+    }
+
+    this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs)
+  }
+
+  disable () {
+    if (this.firstRunTimeout) {
+      clearTimeout(this.firstRunTimeout)
+      this.firstRunTimeout = undefined
+    }
+
+    if (this.interval) {
+      clearInterval(this.interval)
+      this.interval = undefined
+    }
+  }
+
+  async execute () {
+    if (this.isRunning === true) return
+    this.isRunning = true
+
+    try {
+      await this.internalExecute()
+    } catch (err) {
+      logger.error('Cannot execute ' + this.constructor.name + ' scheduler.', { err, ...lTags() })
+    } finally {
+      this.isRunning = false
+    }
+  }
+
+  protected abstract internalExecute (): Promise<any> | Bluebird<any>
+}

server/core/lib/schedulers/actor-follow-scheduler.ts

@@ -0,0 +1,60 @@
+import { isProdInstance, isTestOrDevInstance } from '@peertube/peertube-node-utils'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { ActorFollowModel } from '../../models/actor/actor-follow.js'
+import { ActorFollowHealthCache } from '../actor-follow-health-cache.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class ActorFollowScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    // Run too often in test/dev instances
+    if (isProdInstance()) {
+      logger.info('Processing pending actor follow scores.', lTags())
+    }
+
+    await this.processPendingScores()
+
+    await this.removeBadActorFollows()
+  }
+
+  private async processPendingScores () {
+    const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore()
+    const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds()
+    const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds()
+
+    ActorFollowHealthCache.Instance.clearPendingFollowsScore()
+    ActorFollowHealthCache.Instance.clearBadFollowingServerIds()
+    ActorFollowHealthCache.Instance.clearGoodFollowingServerIds()
+
+    for (const inbox of Object.keys(pendingScores)) {
+      await ActorFollowModel.updateScore(inbox, pendingScores[inbox])
+    }
+
+    await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY)
+    await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS)
+  }
+
+  private async removeBadActorFollows () {
+    if (!isTestOrDevInstance()) logger.info('Removing bad actor follows (scheduler).', lTags())
+
+    try {
+      await ActorFollowModel.removeBadActorFollows()
+    } catch (err) {
+      logger.error('Error in bad actor follows scheduler.', { err, ...lTags() })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/auto-follow-index-instances.ts

@@ -0,0 +1,74 @@
+import { doJSONRequest } from '@server/helpers/requests.js'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+import { ActorFollowModel } from '@server/models/actor/actor-follow.js'
+import { getServerActor } from '@server/models/application/application.js'
+import chunk from 'lodash-es/chunk.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class AutoFollowIndexInstances extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES
+
+  private lastCheck: Date
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected async internalExecute () {
+    return this.autoFollow()
+  }
+
+  private async autoFollow () {
+    if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return
+
+    const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL
+
+    logger.info(`Auto follow instances of index ${indexUrl}`, lTags())
+
+    try {
+      const serverActor = await getServerActor()
+
+      const searchParams = { count: 1000 }
+      if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() })
+
+      this.lastCheck = new Date()
+
+      const { body } = await doJSONRequest<any>(indexUrl, { searchParams, preventSSRF: false })
+      if (!body.data || Array.isArray(body.data) === false) {
+        logger.error(`Cannot auto follow instances of index ${indexUrl}. Please check the auto follow URL.`, { body, ...lTags() })
+        return
+      }
+
+      const hosts: string[] = body.data.map(o => o.host)
+      const chunks = chunk(hosts, 20)
+
+      for (const chunk of chunks) {
+        const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk)
+
+        for (const unfollowedHost of unfollowedHosts) {
+          const payload = {
+            host: unfollowedHost,
+            name: SERVER_ACTOR_NAME,
+            followerActorId: serverActor.id,
+            isAutoFollow: true
+          }
+
+          JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload })
+        }
+      }
+    } catch (err) {
+      logger.error(`Cannot auto follow hosts of index ${indexUrl}.`, { err, ...lTags() })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/geo-ip-update-scheduler.ts

@@ -0,0 +1,26 @@
+import { GeoIP } from '@server/helpers/geo-ip.js'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class GeoIPUpdateScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected internalExecute () {
+    logger.info('Running GeoIP update scheduler', lTags())
+
+    return GeoIP.Instance.updateDatabases()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/live-video-start-scheduler.ts

@@ -0,0 +1,37 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants.js'
+import { syncLiveVideoStartJob } from '@server/lib/live-videos/live-video-jobs.js'
+import { LIVE_VIDEO_START_LEAD_TIME_MS } from '@server/lib/live-videos/shared.js'
+import { LiveVideoModel } from '@server/models/video/live-video.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'live-video-start')
+const RECOVERY_WINDOW_MS = 5 * 60 * 1000
+
+export class LiveVideoStartScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.LIVE_VIDEO_START
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    logger.debug('Running live video start scheduler', lTags())
+
+    const now = Date.now()
+    const liveVideos = await LiveVideoModel.listForStarting({
+      startBefore: new Date(now + RECOVERY_WINDOW_MS + LIVE_VIDEO_START_LEAD_TIME_MS),
+      endAfter: new Date(now)
+    })
+    
+    for (const liveVideo of liveVideos) {
+      await syncLiveVideoStartJob(liveVideo, { replace: false })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/peertube-version-check-scheduler.ts

@@ -0,0 +1,55 @@
+import { doJSONRequest } from '@server/helpers/requests.js'
+import { ApplicationModel } from '@server/models/application/application.js'
+import { compareSemVer } from '@peertube/peertube-core-utils'
+import { JoinPeerTubeVersions } from '@peertube/peertube-models'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { PEERTUBE_VERSION, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { Notifier } from '../notifier/index.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class PeerTubeVersionCheckScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected async internalExecute () {
+    return this.checkLatestVersion()
+  }
+
+  private async checkLatestVersion () {
+    if (CONFIG.PEERTUBE.CHECK_LATEST_VERSION.ENABLED === false) return
+
+    logger.info('Checking latest PeerTube version.', lTags())
+
+    const { body } = await doJSONRequest<JoinPeerTubeVersions>(CONFIG.PEERTUBE.CHECK_LATEST_VERSION.URL, { preventSSRF: false })
+
+    if (!body?.peertube?.latestVersion) {
+      logger.warn('Cannot check latest PeerTube version: body is invalid.', { body, ...lTags() })
+      return
+    }
+
+    const latestVersion = body.peertube.latestVersion
+    const application = await ApplicationModel.load()
+
+    // Already checked this version
+    if (application.latestPeerTubeVersion === latestVersion) return
+
+    if (compareSemVer(PEERTUBE_VERSION, latestVersion) < 0) {
+      application.latestPeerTubeVersion = latestVersion
+      await application.save()
+
+      Notifier.Instance.notifyOfNewPeerTubeVersion(application, latestVersion)
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/plugins-check-scheduler.ts

@@ -0,0 +1,75 @@
+import { compareSemVer } from '@peertube/peertube-core-utils'
+import chunk from 'lodash-es/chunk.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { PluginModel } from '../../models/server/plugin.js'
+import { Notifier } from '../notifier/index.js'
+import { getLatestPluginsVersion } from '../plugins/plugin-index.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class PluginsCheckScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PLUGINS
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected async internalExecute () {
+    return this.checkLatestPluginsVersion()
+  }
+
+  private async checkLatestPluginsVersion () {
+    if (CONFIG.PLUGINS.INDEX.ENABLED === false) return
+
+    logger.info('Checking latest plugins version.', lTags())
+
+    const plugins = await PluginModel.listInstalled()
+
+    // Process 10 plugins in 1 HTTP request
+    const chunks = chunk(plugins, 10)
+    for (const chunk of chunks) {
+      // Find plugins according to their npm name
+      const pluginIndex: { [npmName: string]: PluginModel } = {}
+      for (const plugin of chunk) {
+        pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin
+      }
+
+      const npmNames = Object.keys(pluginIndex)
+
+      try {
+        const results = await getLatestPluginsVersion(npmNames)
+
+        for (const result of results) {
+          const plugin = pluginIndex[result.npmName]
+          if (!result.latestVersion) continue
+
+          if (
+            !plugin.latestVersion ||
+            (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0)
+          ) {
+            plugin.latestVersion = result.latestVersion
+            await plugin.save()
+
+            // Notify if there is an higher plugin version available
+            if (compareSemVer(plugin.version, result.latestVersion) < 0) {
+              Notifier.Instance.notifyOfNewPluginVersion(plugin)
+            }
+
+            logger.info(`Plugin ${result.npmName} has a new latest version ${plugin.latestVersion}`, lTags())
+          }
+        }
+      } catch (err) {
+        logger.error('Cannot get latest plugins version.', { npmNames, err, ...lTags() })
+      }
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts

@@ -0,0 +1,38 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants.js'
+import { uploadx } from '../uploadx.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'resumable-upload', 'cleaner')
+
+export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+  private lastExecutionTimeMs: number
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+
+    this.lastExecutionTimeMs = new Date().getTime()
+  }
+
+  protected async internalExecute () {
+    logger.info('Removing dangling resumable uploads', lTags())
+
+    const now = new Date().getTime()
+
+    try {
+      // Remove files that were not updated since the last execution
+      await uploadx.storage.purge(now - this.lastExecutionTimeMs)
+    } catch (error) {
+      logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
+    } finally {
+      this.lastExecutionTimeMs = now
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/remove-expired-user-exports-scheduler.ts

@@ -0,0 +1,33 @@
+import { UserExportModel } from '@server/models/user/user-export.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class RemoveExpiredUserExportsScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_EXPIRED_USER_EXPORTS
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    logger.info('Running expired user exports checker.', lTags())
+
+    const expired = await UserExportModel.listExpired(CONFIG.EXPORT.USERS.EXPORT_EXPIRATION)
+
+    for (const userExport of expired) {
+      logger.info(`Removing expired user exports ${userExport.filename}`, lTags())
+
+      await userExport.destroy()
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/remove-old-history-scheduler.ts

@@ -0,0 +1,32 @@
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { UserVideoHistoryModel } from '../../models/user/user-video-history.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class RemoveOldHistoryScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected internalExecute () {
+    if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
+
+    logger.info('Removing old videos history.', lTags())
+
+    const now = new Date()
+    const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
+
+    return UserVideoHistoryModel.removeOldHistory(beforeDate)
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/remove-old-views-scheduler.ts

@@ -0,0 +1,50 @@
+import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js'
+import { VideoViewModel } from '@server/models/view/video-view.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers')
+
+export class RemoveOldViewsScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected async internalExecute () {
+    await this.removeRemoteViews()
+    await this.removeLocalViews()
+  }
+
+  private removeRemoteViews () {
+    if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE <= 0) return
+
+    logger.info('Removing old views from remote videos.', lTags())
+
+    const now = new Date()
+    const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
+
+    return VideoViewModel.removeOldRemoteViews(beforeDate)
+  }
+
+  private async removeLocalViews () {
+    if (CONFIG.VIEWS.VIDEOS.LOCAL.MAX_AGE <= 0) return
+
+    logger.info('Removing old views from local videos.', lTags())
+
+    const now = new Date()
+    const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.LOCAL.MAX_AGE).toISOString()
+
+    await VideoViewModel.removeOldLocalViews(beforeDate)
+    await LocalVideoViewerModel.removeOldViews(beforeDate)
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/runner-job-watch-dog-scheduler.ts

@@ -0,0 +1,55 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { RunnerJobModel } from '@server/models/runner/runner-job.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { getRunnerJobHandlerClass } from '../runners/index.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'runner')
+
+export class RunnerJobWatchDogScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    const vodStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD,
+      types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]
+    })
+
+    const liveStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE,
+      types: [ 'live-rtmp-hls-transcoding' ]
+    })
+
+    const transcriptionStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.TRANSCRIPTION,
+      types: [ 'video-transcription' ]
+    })
+
+    const studioStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.STUDIO,
+      types: [ 'video-studio-transcoding' ]
+    })
+
+    for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs, ...transcriptionStalledJobs, ...studioStalledJobs ]) {
+      logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type))
+
+      const Handler = getRunnerJobHandlerClass(stalled)
+
+      await new Handler().abort({
+        runnerJob: stalled,
+        abortNotSupportedErrorMessage: 'Stalled runner job'
+      })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/update-token-session-scheduler.ts

@@ -0,0 +1,53 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { OAuthTokenModel } from '@server/models/oauth/oauth-token.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+import { isProdInstance } from '@peertube/peertube-node-utils'
+
+const lTags = loggerTagsFactory('schedulers')
+
+type UpdatePayload = {
+  lastActivityDate: Date
+  lastActivityIP: string
+  lastActivityDevice: string
+}
+
+export class UpdateTokenSessionScheduler extends AbstractScheduler {
+  private static instance: UpdateTokenSessionScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_TOKEN_SESSION
+
+  private toUpdate = new Map<number, UpdatePayload>()
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  addToUpdate (id: number, payload: UpdatePayload) {
+    this.toUpdate.set(id, payload)
+  }
+
+  protected async internalExecute () {
+    // Log only on production instances to reduce noise on development/test instances
+    if (isProdInstance()) logger.debug('Running update token session scheduler', lTags())
+
+    const entriesToUpdate = this.toUpdate.entries()
+    this.toUpdate = new Map()
+
+    for (const [ id, payload ] of entriesToUpdate) {
+      await OAuthTokenModel.update({
+        lastActivityDate: payload.lastActivityDate,
+        lastActivityIP: payload.lastActivityIP,
+        lastActivityDevice: payload.lastActivityDevice
+      }, {
+        where: { id },
+        // Prevent tokens cache invalidation, we don't update fields that are meaningful for this cache
+        hooks: false
+      })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/update-videos-scheduler.ts

@@ -0,0 +1,95 @@
+import { VideoPrivacy, VideoPrivacyType, VideoState } from '@peertube/peertube-models'
+import { VideoModel } from '@server/models/video/video.js'
+import { MScheduleVideoUpdate } from '@server/types/models/index.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { sequelizeTypescript } from '../../initializers/database.js'
+import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update.js'
+import { isNewVideoPrivacyForFederation } from '../activitypub/videos/federate.js'
+import { Notifier } from '../notifier/index.js'
+import { addVideoJobsAfterUpdate } from '../video-jobs.js'
+import { VideoPathManager } from '../video-path-manager.js'
+import { setVideoPrivacy } from '../video-privacy.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'update-videos')
+
+export class UpdateVideosScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    return this.updateVideos()
+  }
+
+  private async updateVideos () {
+    logger.debug('Running update videos scheduler', lTags())
+
+    if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined
+
+    const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate()
+
+    for (const schedule of schedules) {
+      const videoOnly = await VideoModel.load(schedule.videoId)
+      if (!videoOnly) continue
+
+      const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid)
+
+      try {
+        const { video, published } = await this.updateAVideo(schedule)
+
+        if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video)
+      } catch (err) {
+        logger.error('Cannot update video ' + videoOnly.uuid, { err, ...lTags(videoOnly.uuid) })
+      }
+
+      mutexReleaser()
+    }
+  }
+
+  private async updateAVideo (schedule: MScheduleVideoUpdate) {
+    let oldPrivacy: VideoPrivacyType
+    let isNewVideoForFederation: boolean
+    let published = false
+
+    const video = await sequelizeTypescript.transaction(async t => {
+      const video = await VideoModel.loadFull(schedule.videoId, t)
+      if (video.state === VideoState.TO_TRANSCODE) return null
+
+      logger.info('Executing scheduled video update on ' + video.uuid, lTags(video.uuid))
+
+      if (schedule.privacy) {
+        isNewVideoForFederation = isNewVideoPrivacyForFederation(video.privacy, schedule.privacy)
+        oldPrivacy = video.privacy
+
+        setVideoPrivacy(video, schedule.privacy)
+        await video.save({ transaction: t })
+
+        if (oldPrivacy === VideoPrivacy.PRIVATE) {
+          published = true
+        }
+      }
+
+      await schedule.destroy({ transaction: t })
+
+      return video
+    })
+
+    if (!video) {
+      return { video, published: false }
+    }
+
+    await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideoForFederation, nameChanged: false })
+
+    return { video, published }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts

@@ -0,0 +1,70 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js'
+import { VideoChannelModel } from '@server/models/video/video-channel.js'
+import { VideoImportModel } from '@server/models/video/video-import.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { synchronizeChannel } from '../sync-channel.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'channel-synchronization')
+
+export class VideoChannelSyncLatestScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected async internalExecute () {
+    if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) {
+      logger.debug('Discard channels synchronization as the feature is disabled', lTags())
+      return
+    }
+
+    logger.info('Checking channels to synchronize', lTags())
+
+    const channelSyncs = await VideoChannelSyncModel.listSyncs()
+
+    for (const sync of channelSyncs) {
+      const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId)
+
+      // We can't rely on publication date for playlist elements
+      // For example, an old video may have been added to a playlist since the last sync
+      let skipPublishedBeforeOrEq: Date
+
+      if (!this.isPlaylistUrl(sync.externalChannelUrl)) {
+        const lastImport = await VideoImportModel.loadLastImportBySyncId({ channelSyncId: sync.id })
+
+        if (lastImport && lastImport.Video?.originallyPublishedAt) {
+          skipPublishedBeforeOrEq = lastImport.Video.originallyPublishedAt
+        } else {
+          skipPublishedBeforeOrEq = sync.lastSyncAt || sync.createdAt
+        }
+      }
+
+      await synchronizeChannel({
+        channel,
+        externalChannelUrl: sync.externalChannelUrl,
+        videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION,
+        channelSync: sync,
+        skipPublishedBeforeOrEq
+      })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  private isPlaylistUrl (url: string): boolean {
+    const parsed = new URL(url)
+    const pathname = parsed.pathname.toLowerCase()
+
+    return pathname.startsWith('/playlist/') || // Dailymotion playlist
+      pathname.startsWith('/showcase/') || // Vimeo playlist
+      pathname === '/playlist' || // YouTube playlist
+      pathname.startsWith('/w/p/') // PeerTube playlist
+  }
+}

server/core/lib/schedulers/video-views-buffer-scheduler.ts

@@ -0,0 +1,53 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { federateVideoIfNeeded } from '../activitypub/videos/index.js'
+import { Redis } from '../redis.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'views')
+
+export class VideoViewsBufferScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.VIDEO_VIEWS_BUFFER_UPDATE
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    logger.debug('Running video views buffer scheduler', lTags())
+
+    const videoIds = await Redis.Instance.listLocalVideosViewed()
+    if (videoIds.length === 0) return
+
+    for (const videoId of videoIds) {
+      try {
+        const views = await Redis.Instance.getLocalVideoViews(videoId)
+        await Redis.Instance.deleteLocalVideoViews(videoId)
+
+        const video = await VideoModel.loadFull(videoId)
+        if (!video) {
+          logger.debug('Video %d does not exist anymore, skipping videos view addition.', videoId, lTags())
+          continue
+        }
+
+        logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid))
+
+        // If this is a remote video, the origin instance will send us an update
+        await VideoModel.incrementViews(videoId, views)
+
+        // Send video update
+        video.views += views
+        await federateVideoIfNeeded(video, false)
+      } catch (err) {
+        logger.error('Cannot process local video views buffer of video %d.', videoId, { err, ...lTags() })
+      }
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}

server/core/lib/schedulers/videos-redundancy-scheduler.ts

@@ -0,0 +1,314 @@
+import { VideosRedundancyStrategy } from '@peertube/peertube-models'
+import { getServerActor } from '@server/models/application/application.js'
+import { VideoModel } from '@server/models/video/video.js'
+import {
+  MStreamingPlaylistFiles,
+  MVideoAccountLight,
+  MVideoFile,
+  MVideoRedundancyStreamingPlaylistVideo,
+  MVideoRedundancyVideo,
+  MVideoWithAllFiles
+} from '@server/types/models/index.js'
+import { join } from 'path'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { CONFIG } from '../../initializers/config.js'
+import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants.js'
+import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js'
+import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send/index.js'
+import { getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url.js'
+import { getOrCreateAPVideo } from '../activitypub/videos/index.js'
+import { downloadPlaylistSegments } from '../hls.js'
+import { removeVideoRedundancy } from '../redundancy.js'
+import { generateHLSRedundancyUrl } from '../video-urls.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'redundancy')
+
+type CandidateToDuplicate = {
+  redundancy: VideosRedundancyStrategy
+  video: MVideoWithAllFiles
+  streamingPlaylists: MStreamingPlaylistFiles[]
+}
+
+export class VideosRedundancyScheduler extends AbstractScheduler {
+  private static instance: VideosRedundancyScheduler
+
+  protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  async createManualRedundancy (videoId: number) {
+    const videoToDuplicate = await VideoModel.loadWithFiles(videoId)
+
+    if (!videoToDuplicate) {
+      logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags())
+      return
+    }
+
+    return this.createVideoRedundancies({
+      video: videoToDuplicate,
+      redundancy: null,
+      streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
+    })
+  }
+
+  protected async internalExecute () {
+    for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
+      logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags())
+
+      try {
+        const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig)
+        if (!videoToDuplicate) continue
+
+        const candidateToDuplicate = {
+          video: videoToDuplicate,
+          redundancy: redundancyConfig,
+          files: videoToDuplicate.VideoFiles,
+          streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
+        }
+
+        await this.purgeCacheIfNeeded(candidateToDuplicate)
+
+        if (await this.isTooHeavy(candidateToDuplicate)) {
+          logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid))
+          continue
+        }
+
+        logger.info(
+          'Will duplicate video %s in redundancy scheduler "%s".',
+          videoToDuplicate.url,
+          redundancyConfig.strategy,
+          lTags(videoToDuplicate.uuid)
+        )
+
+        await this.createVideoRedundancies(candidateToDuplicate)
+      } catch (err) {
+        logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() })
+      }
+    }
+
+    await this.extendsLocalExpiration()
+
+    await this.purgeRemoteExpired()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  private async extendsLocalExpiration () {
+    const expired = await VideoRedundancyModel.listLocalExpired()
+
+    for (const redundancyModel of expired) {
+      try {
+        const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
+
+        // If the admin disabled the redundancy, remove this redundancy instead of extending it
+        if (!redundancyConfig) {
+          logger.info(
+            'Destroying redundancy %s because the redundancy %s does not exist anymore.',
+            redundancyModel.url,
+            redundancyModel.strategy
+          )
+
+          await removeVideoRedundancy(redundancyModel)
+          continue
+        }
+
+        const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy)
+
+        // If the admin decreased the cache size, remove this redundancy instead of extending it
+        if (totalUsed > redundancyConfig.size) {
+          logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy)
+
+          await removeVideoRedundancy(redundancyModel)
+          continue
+        }
+
+        await this.extendsRedundancy(redundancyModel)
+      } catch (err) {
+        logger.error(
+          'Cannot extend or remove expiration of %s video from our redundancy system.',
+          this.buildEntryLogId(redundancyModel),
+          { err, ...lTags(redundancyModel.getVideoUUID()) }
+        )
+      }
+    }
+  }
+
+  private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) {
+    const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
+    // Redundancy strategy disabled, remove our redundancy instead of extending expiration
+    if (!redundancy) {
+      await removeVideoRedundancy(redundancyModel)
+      return
+    }
+
+    await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
+  }
+
+  private async purgeRemoteExpired () {
+    const expired = await VideoRedundancyModel.listRemoteExpired()
+
+    for (const redundancyModel of expired) {
+      try {
+        await removeVideoRedundancy(redundancyModel)
+      } catch (err) {
+        logger.error(
+          'Cannot remove redundancy %s from our redundancy system.',
+          this.buildEntryLogId(redundancyModel),
+          lTags(redundancyModel.getVideoUUID())
+        )
+      }
+    }
+  }
+
+  private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
+    if (cache.strategy === 'most-views') {
+      return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    }
+
+    if (cache.strategy === 'trending') {
+      return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    }
+
+    if (cache.strategy === 'recently-added') {
+      const minViews = cache.minViews
+      return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
+    }
+  }
+
+  private async createVideoRedundancies (data: CandidateToDuplicate) {
+    const video = await this.loadAndRefreshVideo(data.video.url)
+
+    if (!video) {
+      logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid))
+
+      return
+    }
+
+    // Only HLS player supports redundancy, so do not duplicate web videos
+    for (const streamingPlaylist of data.streamingPlaylists) {
+      const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id)
+      if (existingRedundancy) {
+        await this.extendsRedundancy(existingRedundancy)
+
+        continue
+      }
+
+      await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist)
+    }
+  }
+
+  private async createStreamingPlaylistRedundancy (
+    redundancy: VideosRedundancyStrategy,
+    video: MVideoAccountLight,
+    playlistArg: MStreamingPlaylistFiles
+  ) {
+    let strategy = 'manual'
+    let expiresOn: Date = null
+
+    if (redundancy) {
+      strategy = redundancy.strategy
+      expiresOn = this.buildNewExpiration(redundancy.minLifetime)
+    }
+
+    const playlist = Object.assign(playlistArg, { Video: video })
+    const serverActor = await getServerActor()
+
+    logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid))
+
+    const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid)
+    const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video)
+
+    const maxSizeKB = this.getTotalFileSizes([ playlist ]) / 1000
+    const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance
+    await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB)
+
+    const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
+      expiresOn,
+      url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
+      fileUrl: generateHLSRedundancyUrl(video, playlistArg),
+      strategy,
+      videoStreamingPlaylistId: playlist.id,
+      actorId: serverActor.id
+    })
+
+    createdModel.VideoStreamingPlaylist = playlist
+
+    await sendCreateCacheFile(serverActor, video, createdModel)
+
+    logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid))
+  }
+
+  private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) {
+    logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID()))
+
+    const serverActor = await getServerActor()
+
+    redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs)
+    await redundancy.save()
+
+    await sendUpdateCacheFile(serverActor, redundancy)
+  }
+
+  private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) {
+    while (await this.isTooHeavy(candidateToDuplicate)) {
+      const redundancy = candidateToDuplicate.redundancy
+      const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime)
+      if (!toDelete) return
+
+      const redundancies = await VideoRedundancyModel.listLocalByStreamingPlaylistId(toDelete.VideoStreamingPlaylist.id)
+
+      for (const redundancy of redundancies) {
+        await removeVideoRedundancy(redundancy)
+      }
+    }
+  }
+
+  private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) {
+    const maxSize = candidateToDuplicate.redundancy.size
+
+    const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy)
+
+    const videoSize = this.getTotalFileSizes(candidateToDuplicate.streamingPlaylists)
+    const willUse = alreadyUsed + videoSize
+
+    logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) })
+
+    return willUse > maxSize
+  }
+
+  private buildNewExpiration (expiresAfterMs: number) {
+    return new Date(Date.now() + expiresAfterMs)
+  }
+
+  private buildEntryLogId (object: MVideoRedundancyStreamingPlaylistVideo) {
+    return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}`
+  }
+
+  private getTotalFileSizes (playlists: MStreamingPlaylistFiles[]): number {
+    const fileReducer = (previous: number, current: MVideoFile) => previous + current.size
+
+    let allFiles: MVideoFile[] = []
+    for (const p of playlists) {
+      allFiles = allFiles.concat(p.VideoFiles)
+    }
+
+    return allFiles.reduce(fileReducer, 0)
+  }
+
+  private async loadAndRefreshVideo (videoUrl: string) {
+    // We need more attributes and check if the video still exists
+    const getVideoOptions = {
+      videoObject: videoUrl,
+      syncParam: { rates: false, shares: false, comments: false, refreshVideo: true },
+      fetchType: 'all' as 'all'
+    }
+    const { video } = await getOrCreateAPVideo(getVideoOptions)
+
+    return video
+  }
+}

server/core/lib/schedulers/youtube-dl-update-scheduler.ts

@@ -0,0 +1,26 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { YoutubeDLCLI } from '@server/helpers/youtube-dl/index.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'youtube-dl')
+
+export class YoutubeDlUpdateScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE
+
+  private constructor () {
+    super({ randomRunOnEnable: true })
+  }
+
+  protected internalExecute () {
+    logger.info('Running youtube-dl updated scheduler', lTags())
+
+    return YoutubeDLCLI.updateYoutubeDLBinary()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}