Init commit

server/core/helpers/youtube-dl/youtube-dl-wrapper.ts

@@ -0,0 +1,248 @@
+import { VideoResolutionType } from '@peertube/peertube-models'
+import { CONFIG } from '@server/initializers/config.js'
+import { ExecaError } from 'execa'
+import { move, pathExists, remove } from 'fs-extra/esm'
+import { readdir } from 'fs/promises'
+import { dirname, join } from 'path'
+import { inspect } from 'util'
+import { isVideoFileExtnameValid } from '../custom-validators/videos.js'
+import { t } from '../i18n.js'
+import { logger, loggerTagsFactory } from '../logger.js'
+import { generateVideoImportTmpPath } from '../utils.js'
+import { YoutubeDLCLI } from './youtube-dl-cli.js'
+import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder.js'
+
+const lTags = loggerTagsFactory('youtube-dl')
+
+export const YoutubeDlImportErrorCode = {
+  FETCH_ERROR: 0,
+  NOT_ONLY_UNICAST_URL: 1,
+  SKIP_PUBLICATION_DATE: 2,
+  IS_LIVE: 3
+}
+
+export type YoutubeDlImportErrorCodeType = typeof YoutubeDlImportErrorCode[keyof typeof YoutubeDlImportErrorCode]
+
+export class YoutubeDlImportError extends Error {
+  static fromError (options: {
+    err: Error
+    code: YoutubeDlImportErrorCodeType
+    message?: string
+  }) {
+    const { err, code, message } = options
+
+    const ytDlErr = new this({ message: message ?? err.message, code })
+    ytDlErr.cause = err
+
+    return ytDlErr
+  }
+
+  code: YoutubeDlImportErrorCodeType
+
+  constructor ({ message, code }) {
+    super(message)
+
+    this.code = code
+  }
+
+  isUnavailableVideoError () {
+    const stderr = this.getStderr()
+
+    if (stderr.includes('Video unavailable') || stderr.includes(' 429 ')) {
+      return true
+    }
+
+    return false
+  }
+
+  isRateLimitError () {
+    const stderr = this.getStderr()
+
+    if (stderr.includes('Sign in to confirm you’re not a bot')) {
+      return true
+    }
+
+    return false
+  }
+
+  private getStderr () {
+    const stderr = (this.cause as ExecaError)?.stderr
+
+    if (typeof stderr === 'string') return stderr
+
+    return ''
+  }
+}
+
+export type YoutubeDLSubs = {
+  language: string
+  filename: string
+  path: string
+}[]
+
+const processOptions = {
+  maxBuffer: 1024 * 1024 * 30 // 30MB
+}
+
+export class YoutubeDLWrapper {
+  constructor (
+    private readonly url: string,
+    private readonly enabledResolutions: VideoResolutionType[],
+    private readonly useBestFormat: boolean
+  ) {
+  }
+
+  async getInfoForDownload (options: {
+    userLanguage: string
+    youtubeDLArgs?: string[]
+  }): Promise<YoutubeDLInfo> {
+    const { userLanguage, youtubeDLArgs = [] } = options
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    try {
+      const info = await youtubeDL.getInfo({
+        url: this.url,
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
+        additionalYoutubeDLArgs: youtubeDLArgs,
+        processOptions
+      })
+
+      if (!info) {
+        throw new YoutubeDlImportError({
+          message: t(`Cannot fetch information from import for URL {targetUrl}`, userLanguage, { targetUrl: this.url }),
+          code: YoutubeDlImportErrorCode.FETCH_ERROR
+        })
+      }
+
+      if (info.is_live === true || [ 'is_live', 'post_live', 'is_upcoming' ].includes(info.live_status)) {
+        throw new YoutubeDlImportError({
+          message: t('Cannot download a live streaming for URL {targetUrl}', userLanguage, { targetUrl: this.url }),
+          code: YoutubeDlImportErrorCode.IS_LIVE
+        })
+      }
+
+      const infoBuilder = new YoutubeDLInfoBuilder(info)
+
+      return infoBuilder.getInfo()
+    } catch (err) {
+      throw YoutubeDlImportError.fromError({
+        err,
+        message: t(`Cannot get info from {targetUrl}`, userLanguage, { targetUrl: this.url }),
+        code: YoutubeDlImportErrorCode.FETCH_ERROR
+      })
+    }
+  }
+
+  async getInfoForListImport (options: {
+    userLanguage: string
+    latestVideosCount?: number
+  }) {
+    const { userLanguage } = options
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const list = await youtubeDL.getListInfo({
+      url: this.url,
+      latestVideosCount: options.latestVideosCount,
+      processOptions
+    })
+
+    if (!Array.isArray(list)) {
+      logger.debug(`List info from youtube-dl is not an array for ${this.url}.`, { info: inspect(list), ...lTags() })
+
+      throw new YoutubeDlImportError({
+        message: t(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`, userLanguage, { targetUrl: this.url }),
+        code: YoutubeDlImportErrorCode.FETCH_ERROR
+      })
+    }
+
+    return list.map(info => info.webpage_url)
+  }
+
+  async getSubtitles (): Promise<YoutubeDLSubs> {
+    const cwd = CONFIG.STORAGE.TMP_DIR
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } })
+    if (!files) return []
+
+    logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() })
+
+    const subtitles = files.reduce((acc, filename) => {
+      const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i)
+      if (!matched?.[1]) return acc
+
+      return [
+        ...acc,
+        {
+          language: matched[1],
+          path: join(cwd, filename),
+          filename
+        }
+      ]
+    }, [])
+
+    return subtitles
+  }
+
+  async downloadVideo (fileExt: string, timeout: number): Promise<string> {
+    // Leave empty the extension, youtube-dl will add it
+    const pathWithoutExtension = generateVideoImportTmpPath(this.url, '')
+
+    logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags())
+
+    const youtubeDL = await YoutubeDLCLI.safeGet()
+
+    try {
+      await youtubeDL.download({
+        url: this.url,
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat),
+        output: pathWithoutExtension,
+        timeout,
+        processOptions
+      })
+
+      // If youtube-dl did not guess an extension for our file, just use .mp4 as default
+      if (await pathExists(pathWithoutExtension)) {
+        await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
+      }
+
+      const path = await this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+      if (!path) {
+        const directoryContent = await readdir(dirname(pathWithoutExtension))
+
+        throw new Error(`Cannot guess path of ${pathWithoutExtension}. Directory content: ${directoryContent.join(', ')}`)
+      }
+
+      return path
+    } catch (err) {
+      this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+        .then(path => {
+          logger.debug('Error in youtube-dl import, deleting file if exists.', { err, pathToDelete: path, ...lTags() })
+
+          if (path) return remove(path)
+        })
+        .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() }))
+
+      throw err
+    }
+  }
+
+  private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {
+    if (!isVideoFileExtnameValid(sourceExt)) {
+      throw new Error('Invalid video extension ' + sourceExt)
+    }
+
+    const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ]
+
+    for (const extension of extensions) {
+      const path = tmpPath + extension
+
+      if (await pathExists(path)) return path
+    }
+
+    return undefined
+  }
+}