Init commit

packages/tests/src/transcription/levenshtein-distance.spec.ts

@@ -0,0 +1,18 @@
+import { levenshteinDistance } from '@peertube/peertube-transcription-devtools'
+import { expect } from 'chai'
+
+describe('Levenshtein distance', function () {
+  it(`equals 1 when there is only one character difference`, function () {
+    expect(levenshteinDistance('abcd', 'abce')).equals(1)
+  })
+
+  it(`may calculate a distance on a txt subtitle content `, function () {
+    expect(levenshteinDistance(`December, 1965.
+Is that all it has been since
+I inherited the world?
+Only three years.
+Seems like a hundred million.
+
+`, 'December 1965, is that all it has been since I inherited the world only three years, seems like a hundred million.')).equals(13)
+  })
+})

packages/tests/src/transcription/subtitle.spec.ts

@@ -0,0 +1,33 @@
+import { srtToTxt } from '@peertube/peertube-transcription'
+import { expect } from 'chai'
+
+describe('srt to txt', function () {
+  it(`Transforms the content of a srt subtitle to a pure text version`, function () {
+    const txt = srtToTxt(`1
+00:00:00,000 --> 00:00:01,940
+December, 1965.
+
+2
+00:00:03,460 --> 00:00:06,660
+Is that all it has been since
+I inherited the world?
+
+3
+00:00:07,020 --> 00:00:08,900
+Only three years.
+
+4
+00:00:09,940 --> 00:00:11,760
+Seems like a hundred million.
+
+`)
+
+    expect(txt).equals(`December, 1965.
+Is that all it has been since
+I inherited the world?
+Only three years.
+Seems like a hundred million.
+
+`)
+  })
+})

packages/tests/src/transcription/transcriber-factory.spec.ts

@@ -0,0 +1,15 @@
+import { TranscriptionEngineName, transcriberFactory } from '@peertube/peertube-transcription'
+import { createConsoleLogger } from '@tests/shared/common.js'
+
+describe('Transcriber factory', function () {
+  const transcribers: TranscriptionEngineName[] = [ 'openai-whisper', 'whisper-ctranslate2' ]
+
+  describe('Should be able to create a transcriber for each available transcription engine', function () {
+
+    for (const transcriberName of transcribers) {
+      it(`Should be able to create a(n) ${transcriberName} transcriber`, function () {
+        transcriberFactory.createFromEngineName({ engineName: transcriberName, logger: createConsoleLogger() })
+      })
+    }
+  })
+})

packages/tests/src/transcription/transcript/transcript-file-evaluator.spec.ts

@@ -0,0 +1,68 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions, no-new, max-len */
+import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
+import { TranscriptFile } from '@peertube/peertube-transcription'
+import { TranscriptFileEvaluator } from '@peertube/peertube-transcription-devtools'
+import { expect } from 'chai'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+describe('Transcript File Evaluator', function () {
+  const transcriptDirectory = join(tmpdir(), 'peertube-transcription', 'transcript-file-evaluator')
+  const referenceTranscriptFilePath = buildAbsoluteFixturePath('transcription/videos/communiquer-lors-dune-classe-transplantee.txt')
+
+  before(async function () {
+    await ensureDir(transcriptDirectory)
+  })
+
+  it(`may not compare files in another format than txt`, async function () {
+    const vttReference = await TranscriptFile.write({
+      path: join(transcriptDirectory, 'reference.vtt'),
+      format: 'vtt',
+      content: ''
+    })
+    const vttHypothesis = await TranscriptFile.write({
+      path: join(transcriptDirectory, 'hypothesis.vtt'),
+      format: 'vtt',
+      content: ''
+    })
+    expect(() => new TranscriptFileEvaluator(vttReference, vttHypothesis)).to.throw('Can only evaluate txt transcript file')
+  })
+
+  it(`evaluation must return coherent wer & cer`, async function () {
+    const reference = new TranscriptFile({
+      path: referenceTranscriptFilePath,
+      language: 'fr',
+      format: 'txt'
+    })
+    const hypothesis = await TranscriptFile.write({
+      path: join(transcriptDirectory, 'openai.txt'),
+      content: `Communiquez lors d'une classe transplante. Utilisez les photos prises lors de cette classe pour raconter quotidiennement le séjour vécu.
+C'est le scénario P-Dagujic présenté par monsieur Navoli, professeur ainsi que le 3 sur une école alimentaire de Montpellier.
+La première application a utilisé ce ralame déatec. L'enseignant va alors transférer les différentes photos réalisés lors de la classe transplante.
+Dans un dossier, spécifique pour que les élèves puissent le retrouver plus facilement. Il téléverse donc ses photos dans le dossier, dans le venté, dans la médiatèque de la classe.
+Pour terminer, il s'assure que le dossier soit bien ouvert aux utilisateurs afin que tout le monde puisse l'utiliser.
+Les élèves par la suite utilisera le blog. A partir de leurs nantes, il pourront se loi de parposte rédigeant un article d'un reinté.
+Ils illustront ses articles à l'aide des photos de que mon numérique mise à n'accélier dans le venté.
+Pour se faire, il pourront utiliser les diteurs avancés qui les renvèrent directement dans la médiatèque de la classe où il pourront retrouver le dossier créé par leurs enseignants.
+Une fois leur article terminée, les élèves soumétront se lui-ci au professeur qui pourra soit la noté pour correction ou le public.
+Ensuite, il pourront lire et commenter ce de leurs camarades ou répondre aux commentaires de la veille.
+`,
+      format: 'txt',
+      language: 'fr'
+    })
+    const evaluator = new TranscriptFileEvaluator(reference, hypothesis)
+    const wer = await evaluator.wer()
+    expect(wer).to.be.greaterThan(0 / 100)
+    expect(wer).to.be.below(30 / 100)
+
+    const cer = await evaluator.cer()
+    expect(cer).to.be.greaterThan(9 / 100)
+    expect(cer).to.be.below(10 / 100)
+    console.log(await evaluator.alignment())
+  })
+
+  after(async function () {
+    await remove(transcriptDirectory)
+  })
+})

packages/tests/src/transcription/transcript/transcript-file.spec.ts

@@ -0,0 +1,44 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
+import { TranscriptFile } from '@peertube/peertube-transcription'
+import { expect } from 'chai'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+describe('Transcript File', function () {
+  const transcriptFileDirectory = join(tmpdir(), 'peertube-transcription', 'transcript-file')
+
+  before(async function () {
+    await ensureDir(transcriptFileDirectory)
+  })
+
+  it(`may creates a new transcript file from scratch`, async function () {
+    const transcript1 = await TranscriptFile.write({
+      path: join(transcriptFileDirectory, 'test1.txt'),
+      content: 'test2',
+      format: 'txt'
+    })
+    const transcript2 = await TranscriptFile.write({
+      path: join(transcriptFileDirectory, 'test2.txt'),
+      content: 'test2',
+      format: 'txt'
+    })
+
+    expect(await transcript1.equals(transcript2)).to.be.true
+  })
+
+  it(`may creates a txt transcript file object from a transcript without providing the format explicitly`, function () {
+    TranscriptFile.fromPath(buildAbsoluteFixturePath('transcription/videos/the_last_man_on_earth.srt'), 'en')
+    TranscriptFile.fromPath(buildAbsoluteFixturePath('transcription/videos/the_last_man_on_earth.txt'), 'en')
+  })
+
+  it(`fails when loading a file which is obviously not a transcript`, function () {
+    expect(() => TranscriptFile.fromPath(buildAbsoluteFixturePath('transcription/videos/the_last_man_on_earth.mp4'), 'en'))
+      .to.throw(`Couldn't guess transcript format from extension "mp4". Valid formats are: txt, vtt, srt.`)
+  })
+
+  after(async function () {
+    await remove(transcriptFileDirectory)
+  })
+})

packages/tests/src/transcription/utils.spec.ts

@@ -0,0 +1,47 @@
+import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
+import { downloadFile, unzip } from '@peertube/peertube-transcription-devtools'
+import { expect } from 'chai'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { cp, lstat } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+describe('downloadFile', function () {
+  const testDirectory = join(tmpdir(), 'peertube-transcription', 'utils')
+
+  before(async function () {
+    await ensureDir(testDirectory)
+  })
+
+  it(`Downloads a file and write it to the disk `, async function () {
+    const filePath = await downloadFile('https://download.cpy.re/peertube/4k_file.txt', testDirectory)
+
+    expect(await lstat(filePath).then(stats => stats.isFile())).equals(true)
+  })
+
+  after(async function () {
+    await remove(testDirectory)
+  })
+})
+
+describe('unzip', function () {
+  const zipFixtureFileName = 'hello_world.zip'
+  const zipFixtureFilePath = buildAbsoluteFixturePath(`transcription/${zipFixtureFileName}`)
+  const testDirectory = join(tmpdir(), 'peertube-transcription', 'utils')
+
+  before(async function () {
+    await ensureDir(testDirectory)
+  })
+
+  it(`Extract zip archive to directory`, async function () {
+    const zipFilePath = join(testDirectory, zipFixtureFileName)
+    await cp(zipFixtureFilePath, zipFilePath)
+    const unzippedDirectory = await unzip(zipFilePath)
+
+    expect(await lstat(unzippedDirectory).then(stats => stats.isDirectory())).equals(true)
+  })
+
+  after(async function () {
+    await remove(testDirectory)
+  })
+})

packages/tests/src/transcription/whisper/openai-transcriber.spec.ts

@@ -0,0 +1,134 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions, max-len */
+import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
+import {
+  OpenaiTranscriber,
+  TranscriptFile,
+  TranscriptionModel,
+  WhisperBuiltinModel
+} from '@peertube/peertube-transcription'
+import { TranscriptFileEvaluator, levenshteinDistance } from '@peertube/peertube-transcription-devtools'
+import { createConsoleLogger } from '@tests/shared/common.js'
+import { downloadCustomModelsIfNeeded, getCustomModelPath } from '@tests/shared/transcription.js'
+import { config, expect } from 'chai'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+config.truncateThreshold = 0
+
+describe('Open AI Whisper transcriber', function () {
+  const tmpDirectory = join(tmpdir(), 'peertube-transcription')
+  const transcriptDirectory = join(tmpDirectory, 'transcriber', 'openai')
+
+  const shortVideoPath = buildAbsoluteFixturePath('transcription/videos/the_last_man_on_earth.mp4')
+  const frVideoPath = buildAbsoluteFixturePath('transcription/videos/derive_sectaire.mp4')
+
+  const referenceTranscriptFile = new TranscriptFile({
+    path: buildAbsoluteFixturePath('transcription/videos/derive_sectaire.txt'),
+    language: 'fr',
+    format: 'txt'
+  })
+
+  const transcriber = new OpenaiTranscriber({
+    engine: {
+      name: 'openai-whisper',
+      type: 'binary',
+      command: 'whisper',
+      supportedModelFormats: [ 'PyTorch' ],
+      languageDetection: true,
+      version: ''
+    },
+    logger: createConsoleLogger()
+  })
+  const model = new TranscriptionModel('tiny')
+
+  before(async function () {
+    this.timeout(120000)
+
+    await ensureDir(transcriptDirectory)
+
+    await downloadCustomModelsIfNeeded('tiny.pt')
+  })
+
+  it('Should transcribe a media file and provide a valid path to a transcript file in `vtt` format', async function () {
+    this.timeout(3 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({ mediaFilePath: shortVideoPath, language: 'en', format: 'vtt', model, transcriptDirectory })
+
+    expect(transcript.format).to.equals('vtt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('May produce a transcript file in the `srt` format', async function () {
+    const transcript = await transcriber.transcribe({ mediaFilePath: shortVideoPath, language: 'en', format: 'srt', model, transcriptDirectory })
+
+    expect(transcript.format).to.equals('srt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('May produce a transcript file in the `txt` format', async function () {
+    const transcript = await transcriber.transcribe({ mediaFilePath: shortVideoPath, language: 'en', format: 'txt', model, transcriptDirectory })
+
+    expect(transcript.format).to.equals('txt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+    expect(levenshteinDistance(
+      (await transcript.read()).toString(),
+      'December 1965, is that all it has been since I inherited the world only three years, seems like a hundred million.'
+    )).to.be.below(3)
+  })
+
+  it('May transcribe a media file using a local PyTorch model', async function () {
+    this.timeout(2 * 1000 * 60)
+
+    await transcriber.transcribe({
+      mediaFilePath: shortVideoPath,
+      model: await TranscriptionModel.fromPath(getCustomModelPath('tiny.pt')),
+      language: 'en',
+      format: 'vtt',
+      transcriptDirectory
+    })
+  })
+
+  it('May transcribe a media file in french', async function () {
+    this.timeout(5 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({ mediaFilePath: frVideoPath, language: 'fr', format: 'txt', model, transcriptDirectory })
+
+    expect(transcript.format).to.equals('txt')
+    expect(transcript.language).to.equals('fr')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('Guesses the video language if not provided', async function () {
+    this.timeout(3 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({ mediaFilePath: frVideoPath, model, format: 'vtt', transcriptDirectory })
+
+    expect(transcript.language).to.equals('fr')
+  })
+
+  it('May transcribe a media file in french with small model (can be long)', async function () {
+    this.timeout(6 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({
+      mediaFilePath: frVideoPath,
+      language: 'fr',
+      format: 'txt',
+      model: new WhisperBuiltinModel('small'),
+      transcriptDirectory
+    })
+
+    expect(transcript.language).to.equals('fr')
+
+    const transcriptFileEvaluator = new TranscriptFileEvaluator(referenceTranscriptFile, transcript)
+    const cer = await transcriptFileEvaluator.cer()
+    expect(cer).to.be.below(6 / 100)
+  })
+
+  after(async function () {
+    await remove(transcriptDirectory)
+  })
+})

packages/tests/src/transcription/whisper/whisper-ctranslate2.spec.ts

@@ -0,0 +1,174 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions, max-len */
+import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
+import { Ctranslate2Transcriber, OpenaiTranscriber, TranscriptFile, TranscriptionModel } from '@peertube/peertube-transcription'
+import { TranscriptFileEvaluator, levenshteinDistance } from '@peertube/peertube-transcription-devtools'
+import { createConsoleLogger } from '@tests/shared/common.js'
+import { downloadCustomModelsIfNeeded, getCustomModelPath } from '@tests/shared/transcription.js'
+import { config, expect } from 'chai'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+
+config.truncateThreshold = 0
+
+describe('Whisper CTranslate2 transcriber', function () {
+  const tmpDirectory = join(tmpdir(), 'peertube-transcription')
+  const transcriptDirectory = join(tmpDirectory, 'transcriber', 'ctranslate2')
+
+  const shortVideoPath = buildAbsoluteFixturePath('transcription/videos/the_last_man_on_earth.mp4')
+  const frVideoPath = buildAbsoluteFixturePath('transcription/videos/derive_sectaire.mp4')
+
+  const transcriber = new Ctranslate2Transcriber({
+    engine: {
+      name: 'whisper-ctranslate2',
+      type: 'binary',
+      command: 'whisper-ctranslate2',
+      supportedModelFormats: [ 'CTranslate2' ],
+      languageDetection: true,
+      version: '0.4.4'
+    },
+    logger: createConsoleLogger()
+  })
+
+  const model = new TranscriptionModel('tiny')
+
+  before(async function () {
+    this.timeout(120000)
+
+    await ensureDir(transcriptDirectory)
+
+    await downloadCustomModelsIfNeeded('faster-whisper-tiny')
+  })
+
+  it('Should transcribe a media file and provide a valid path to a transcript file in `vtt` format', async function () {
+    const transcript = await transcriber.transcribe({
+      mediaFilePath: shortVideoPath,
+      language: 'en',
+      format: 'vtt',
+      model,
+      transcriptDirectory
+    })
+
+    expect(transcript.format).to.equals('vtt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('May produce a transcript file in the `srt` format', async function () {
+    const transcript = await transcriber.transcribe({
+      mediaFilePath: shortVideoPath,
+      language: 'en',
+      format: 'srt',
+      model,
+      transcriptDirectory
+    })
+
+    expect(transcript.format).to.equals('srt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('May produce a transcript file in the `txt` format', async function () {
+    const transcript = await transcriber.transcribe({
+      mediaFilePath: shortVideoPath,
+      language: 'en',
+      format: 'txt',
+      model,
+      transcriptDirectory
+    })
+    expect(
+      await transcript.equals(
+        new TranscriptFile({
+          path: join(transcriptDirectory, 'the_last_man_on_earth.txt'),
+          format: 'txt',
+          language: 'en'
+        })
+      )
+    ).to.be.true
+
+    expect(transcript.format).to.equals('txt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+    expect(levenshteinDistance(
+      (await transcript.read()).toString(),
+      'December 1965, is that all it has been since I inherited the world only three years, seems like a hundred million.'
+    )).to.be.below(8)
+  })
+
+  it('May transcribe a media file using a local CTranslate2 model', async function () {
+    this.timeout(2 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({
+      mediaFilePath: shortVideoPath,
+      model: await TranscriptionModel.fromPath(getCustomModelPath('faster-whisper-tiny')),
+      language: 'en',
+      transcriptDirectory,
+      format: 'txt'
+    })
+
+    expect(transcript.format).to.equals('txt')
+    expect(transcript.language).to.equals('en')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('May transcribe a media file in french', async function () {
+    this.timeout(5 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({
+      mediaFilePath: frVideoPath,
+      language: 'fr',
+      format: 'txt',
+      model,
+      transcriptDirectory
+    })
+
+    expect(transcript.format).to.equals('txt')
+    expect(transcript.language).to.equals('fr')
+    expect(await transcript.read()).not.to.be.empty
+  })
+
+  it('Guesses the video language if not provided', async function () {
+    this.timeout(2 * 1000 * 60)
+
+    const transcript = await transcriber.transcribe({ mediaFilePath: frVideoPath, format: 'vtt', model, transcriptDirectory })
+    expect(transcript.language).to.equals('fr')
+  })
+
+  it('Should produce a text transcript similar to openai-whisper implementation', async function () {
+    this.timeout(10 * 1000 * 60)
+
+    const transcribeArgs = {
+      mediaFilePath: frVideoPath,
+      language: 'fr',
+      format: 'txt' as 'txt',
+      transcriptDirectory,
+      model
+    }
+
+    const transcript = await transcriber.transcribe(transcribeArgs)
+
+    const openaiTranscriber = new OpenaiTranscriber({
+      engine: {
+        name: 'openai-whisper',
+        type: 'binary',
+        command: 'whisper',
+        supportedModelFormats: [ 'PyTorch' ],
+        version: '0.4.4'
+      },
+      logger: createConsoleLogger()
+    })
+    const openaiTranscript = await openaiTranscriber.transcribe({
+      ...transcribeArgs,
+
+      transcriptDirectory: join(transcriptDirectory, 'openai-whisper')
+    })
+
+    const transcriptFileEvaluator = new TranscriptFileEvaluator(openaiTranscript, transcript)
+    expect(await transcriptFileEvaluator.wer()).to.be.below(25 / 100)
+    expect(await transcriptFileEvaluator.cer()).to.be.below(10 / 100)
+  })
+
+  after(async function () {
+    await remove(transcriptDirectory)
+  })
+})