Init commit

server/core/lib/schedulers/runner-job-watch-dog-scheduler.ts

@@ -0,0 +1,55 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { RunnerJobModel } from '@server/models/runner/runner-job.js'
+import { logger, loggerTagsFactory } from '../../helpers/logger.js'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js'
+import { getRunnerJobHandlerClass } from '../runners/index.js'
+import { AbstractScheduler } from './abstract-scheduler.js'
+
+const lTags = loggerTagsFactory('schedulers', 'runner')
+
+export class RunnerJobWatchDogScheduler extends AbstractScheduler {
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG
+
+  private constructor () {
+    super({ randomRunOnEnable: false })
+  }
+
+  protected async internalExecute () {
+    const vodStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD,
+      types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]
+    })
+
+    const liveStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE,
+      types: [ 'live-rtmp-hls-transcoding' ]
+    })
+
+    const transcriptionStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.TRANSCRIPTION,
+      types: [ 'video-transcription' ]
+    })
+
+    const studioStalledJobs = await RunnerJobModel.listStalledJobs({
+      staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.STUDIO,
+      types: [ 'video-studio-transcoding' ]
+    })
+
+    for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs, ...transcriptionStalledJobs, ...studioStalledJobs ]) {
+      logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type))
+
+      const Handler = getRunnerJobHandlerClass(stalled)
+
+      await new Handler().abort({
+        runnerJob: stalled,
+        abortNotSupportedErrorMessage: 'Stalled runner job'
+      })
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}