// whisper.cpp aufrufen und die Ausgabe in Woerter mit Zeitstempeln umwandeln. // // Wir nehmen `-ojf` (JSON mit Tokens), nicht die fertige SRT: fuer das // wortweise Hervorheben brauchen wir Zeitstempel je Wort, und die stehen nur // in den Tokens. Tokens sind Wortstuecke - ein neues Wort beginnt dort, wo der // Tokentext mit einem Leerzeichen anfaengt. import fs from 'node:fs/promises'; import path from 'node:path'; import { logger } from '#core/logger.ts'; import { run } from './ffmpeg.ts'; const log = logger('whisper'); export type Word = { text: string; start: number; end: number }; export type WhisperOptions = { binary?: string; model: string; language?: string; threads?: number; timeoutMs?: number; }; /** Sondertokens von whisper.cpp, die nicht in den Text gehoeren. */ const SPECIAL = /^\[_[A-Z]+_?[^\]]*\]$/; type WhisperJson = { transcription?: Array<{ offsets?: { from: number; to: number }; text?: string; tokens?: Array<{ text: string; offsets?: { from: number; to: number } }>; }>; }; export function wordsFromJson(raw: string): Word[] { const data = JSON.parse(raw) as WhisperJson; const words: Word[] = []; for (const seg of data.transcription ?? []) { const tokens = seg.tokens ?? []; if (!tokens.length) { // Kein Token-Detail: dann eben das ganze Segment als ein "Wort" const text = (seg.text ?? '').trim(); if (text && seg.offsets) { words.push({ text, start: seg.offsets.from / 1000, end: seg.offsets.to / 1000 }); } continue; } let current: Word | null = null; for (const t of tokens) { const raw = t.text ?? ''; if (SPECIAL.test(raw.trim())) continue; const startsWord = raw.startsWith(' '); const piece = raw.trim(); if (!piece) continue; const from = (t.offsets?.from ?? 0) / 1000; const to = (t.offsets?.to ?? 0) / 1000; if (!current || startsWord) { if (current) words.push(current); current = { text: piece, start: from, end: to }; } else { current.text += piece; current.end = to; } } if (current) words.push(current); } // Zeitstempel monoton ziehen - vereinzelt liefert whisper.cpp Rueckspruenge for (let i = 1; i < words.length; i++) { const prev = words[i - 1]!; const cur = words[i]!; if (cur.start < prev.end) cur.start = prev.end; if (cur.end <= cur.start) cur.end = cur.start + 0.08; } return words; } export async function transcribe(wav: string, opts: WhisperOptions): Promise { const prefix = wav.replace(/\.wav$/i, ''); const args = [ '-m', opts.model, '-f', wav, '-l', opts.language ?? 'de', '-t', String(opts.threads ?? 4), '-ojf', // JSON mit Tokens '-of', prefix, '-np', // keine Fortschrittsausgabe ]; const started = Date.now(); await run(opts.binary ?? 'whisper-cli', args, { timeoutMs: opts.timeoutMs ?? 15 * 60_000, niceness: 15, }); const jsonFile = `${prefix}.json`; const words = wordsFromJson(await fs.readFile(jsonFile, 'utf8')); log.info(`${words.length} Woerter erkannt in ${((Date.now() - started) / 1000).toFixed(1)} s ` + `(${path.basename(wav)})`); return words; }