abdulive/packages/media/src/whisper.ts

// whisper.cpp aufrufen und die Ausgabe in Woerter mit Zeitstempeln umwandeln.
//
// Wir nehmen `-ojf` (JSON mit Tokens), nicht die fertige SRT: fuer das
// wortweise Hervorheben brauchen wir Zeitstempel je Wort, und die stehen nur
// in den Tokens. Tokens sind Wortstuecke - ein neues Wort beginnt dort, wo der
// Tokentext mit einem Leerzeichen anfaengt.

import fs from 'node:fs/promises';
import path from 'node:path';
import { logger } from '#core/logger.ts';
import { run } from './ffmpeg.ts';

const log = logger('whisper');

export type Word = { text: string; start: number; end: number };

export type WhisperOptions = {
  binary?: string;
  model: string;
  language?: string;
  threads?: number;
  timeoutMs?: number;
};

/** Sondertokens von whisper.cpp, die nicht in den Text gehoeren. */
const SPECIAL = /^\[_[A-Z]+_?[^\]]*\]$/;

type WhisperJson = {
  transcription?: Array<{
    offsets?: { from: number; to: number };
    text?: string;
    tokens?: Array<{ text: string; offsets?: { from: number; to: number } }>;
  }>;
};

export function wordsFromJson(raw: string): Word[] {
  const data = JSON.parse(raw) as WhisperJson;
  const words: Word[] = [];

  for (const seg of data.transcription ?? []) {
    const tokens = seg.tokens ?? [];
    if (!tokens.length) {
      // Kein Token-Detail: dann eben das ganze Segment als ein "Wort"
      const text = (seg.text ?? '').trim();
      if (text && seg.offsets) {
        words.push({ text, start: seg.offsets.from / 1000, end: seg.offsets.to / 1000 });
      }
      continue;
    }

    let current: Word | null = null;
    for (const t of tokens) {
      const raw = t.text ?? '';
      if (SPECIAL.test(raw.trim())) continue;
      const startsWord = raw.startsWith(' ');
      const piece = raw.trim();
      if (!piece) continue;

      const from = (t.offsets?.from ?? 0) / 1000;
      const to = (t.offsets?.to ?? 0) / 1000;

      if (!current || startsWord) {
        if (current) words.push(current);
        current = { text: piece, start: from, end: to };
      } else {
        current.text += piece;
        current.end = to;
      }
    }
    if (current) words.push(current);
  }

  // Zeitstempel monoton ziehen - vereinzelt liefert whisper.cpp Rueckspruenge
  for (let i = 1; i < words.length; i++) {
    const prev = words[i - 1]!;
    const cur = words[i]!;
    if (cur.start < prev.end) cur.start = prev.end;
    if (cur.end <= cur.start) cur.end = cur.start + 0.08;
  }
  return words;
}

export async function transcribe(wav: string, opts: WhisperOptions): Promise<Word[]> {
  const prefix = wav.replace(/\.wav$/i, '');
  const args = [
    '-m', opts.model,
    '-f', wav,
    '-l', opts.language ?? 'de',
    '-t', String(opts.threads ?? 4),
    '-ojf',                 // JSON mit Tokens
    '-of', prefix,
    '-np',                  // keine Fortschrittsausgabe
  ];

  const started = Date.now();
  await run(opts.binary ?? 'whisper-cli', args, {
    timeoutMs: opts.timeoutMs ?? 15 * 60_000,
    niceness: 15,
  });

  const jsonFile = `${prefix}.json`;
  const words = wordsFromJson(await fs.readFile(jsonFile, 'utf8'));
  log.info(`${words.length} Woerter erkannt in ${((Date.now() - started) / 1000).toFixed(1)} s ` +
    `(${path.basename(wav)})`);
  return words;
}