OpenAI Whisper - Speech-to-Text Complete Tutorial

Table of Contents

Introduction

OpenAI Whisper is a general-purpose speech recognition model trained on 680,000 hours of multilingual and multitask supervised data collected from the web. It is capable of multilingual speech recognition, speech translation, and language identification. This tutorial walks you through everything from basic transcription to production-grade deployment with FastAPI.

Prerequisites

• Python 3.9 or higher
• A CUDA-compatible GPU (recommended for larger models)
• Basic familiarity with Python and machine learning concepts

Install the core dependencies:

pip install openai-whisper
pip install faster-whisper
pip install fastapi uvicorn python-multipart
pip install torch torchaudio
pip install datasets transformers

Understanding Whisper Model Sizes

Whisper comes in several model sizes, each offering a tradeoff between speed and accuracy:

| Model | Parameters | English-only | Multilingual | Required VRAM | Relative Speed |

|--------|-----------|--------------|--------------|---------------|----------------|

| tiny | 39 M | tiny.en | tiny | ~1 GB | ~32x |

| base | 74 M | base.en | base | ~1 GB | ~16x |

| small | 244 M | small.en | small | ~2 GB | ~6x |

| medium | 769 M | medium.en | medium | ~5 GB | ~2x |

| large | 1550 M | N/A | large-v3 | ~10 GB | 1x |

Choosing the right model depends on your use case:

import whisper

For quick prototyping or low-resource environments
model = whisper.loadmodel("tiny")


For production with good accuracy
model = whisper.loadmodel("medium")

For best accuracy (requires GPU with 10GB+ VRAM)
model = whisper.loadmodel("large-v3")

import whisper

model = whisper.loadmodel("medium")

Basic transcription
result = model.transcribe("audiosample.mp3")

print(result["text"])

Access detailed segment information
for segment in result["segments"]:
    print(f"[{segment['start']:.2f}s -> {segment['end']:.2f}s] {segment['text']}")

result = model.transcribe(
    "audiosample.mp3",
    language="en",               # Force language (skip detection)
    temperature=0.0,             # Lower = more deterministic
    wordtimestamps=True,        # Get word-level timestamps

    fp16=True,                   # Use half-precision (GPU only)
    beamsize=5,                 # Beam search for better accuracy
    bestof=5,                   # Number of candidates when sampling

    nospeechthreshold=0.6,     # Threshold to filter silent segments

    conditiononprevioustext=True,  # Use context from previous segments
    initialprompt="Technical meeting about machine learning models."

)

With word-level timestamps
for segment in result["segments"]:
    for wordinfo in segment.get("words", []):
        print(f"  {wordinfo['word']} [{wordinfo['start']:.2f}s - {wordinfo['end']:.2f}s]")

import whisper

model = whisper.loadmodel("large-v3")

Automatic language detection
audio = whisper.loadaudio("foreignspeech.mp3")
audio = whisper.padortrim(audio)
mel = whisper.logmelspectrogram(audio).to(model.device)

, probs = model.detectlanguage(mel)
detectedlang = max(probs, key=probs.get)

print(f"Detected language: {detectedlang} (confidence: {probs[detectedlang]:.2%})")


Transcribe in original language
result = model.transcribe("foreignspeech.mp3", language=detectedlang)

print(f"Transcription: {result['text']}")

Translate to English (speech translation)
resulttranslated = model.transcribe(
    "foreignspeech.mp3",

    task="translate"  # Translates any language to English
)
print(f"Translation: {resulttranslated['text']}")

Batch processing multiple files with different languages:

import os
from pathlib import Path

def batchtranscribe(audiodir: str, modelname: str = "medium") -> dict:

    model = whisper.loadmodel(modelname)

    results = {}

    for audiofile in Path(audiodir).glob(".mp3"):

        result = model.transcribe(str(audiofile))
        results[audiofile.name] = {

            "language": result["language"],
            "text": result["text"],
            "segments": result["segments"]
        }
        print(f"Processed: {audiofile.name} (lang: {result['language']})")

    return results

allresults = batchtranscribe("/data/audiofiles/")

from datasets import loaddataset, Audio
from transformers import (
    WhisperFeatureExtractor,
    WhisperTokenizer,
    WhisperProcessor,
    WhisperForConditionalGeneration,
    Seq2SeqTrainingArguments,
    Seq2SeqTrainer,
)
import torch
from dataclasses import dataclass
from typing import Any, Dict, List, Union

Load processor components
featureextractor = WhisperFeatureExtractor.frompretrained("openai/whisper-small")
tokenizer = WhisperTokenizer.frompretrained("openai/whisper-small", language="en", task="transcribe")

processor = WhisperProcessor.frompretrained("openai/whisper-small", language="en", task="transcribe")

Load your custom dataset (example with Common Voice)
dataset = loaddataset("mozilla-foundation/commonvoice110", "en", split="train[:1000]")
dataset = dataset.castcolumn("audio", Audio(samplingrate=16000))

def preparedataset(batch):

    audio = batch["audio"]
    batch["inputfeatures"] = featureextractor(

        audio["array"], samplingrate=audio["samplingrate"]

    ).inputfeatures[0]
    batch["labels"] = tokenizer(batch["sentence"]).inputids

    return batch

dataset = dataset.map(preparedataset, removecolumns=dataset.columnnames, numproc=4)


Data collator for dynamic padding
@dataclass
class DataCollatorSpeechSeq2SeqWithPadding:
    processor: Any
    decoderstarttokenid: int

    def call(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:
        inputfeatures = [{"inputfeatures": f["inputfeatures"]} for f in features]

        batch = self.processor.featureextractor.pad(inputfeatures, returntensors="pt")

        labelfeatures = [{"inputids": f["labels"]} for f in features]
        labelsbatch = self.processor.tokenizer.pad(labelfeatures, returntensors="pt")

        labels = labelsbatch["inputids"].maskedfill(labelsbatch.attentionmask.ne(1), -100)

        if (labels[:, 0] == self.decoderstarttokenid).all().cpu().item():

            labels = labels[:, 1:]

        batch["labels"] = labels
        return batch

Load model and configure training
model = WhisperForConditionalGeneration.frompretrained("openai/whisper-small")
model.generationconfig.language = "en"

model.generationconfig.task = "transcribe"
model.generationconfig.forceddecoderids = None


datacollator = DataCollatorSpeechSeq2SeqWithPadding(
    processor=processor,
    decoderstarttokenid=model.config.decoderstarttokenid,
)

trainingargs = Seq2SeqTrainingArguments(

    outputdir="./whisper-finetuned",
    perdevicetrainbatchsize=16,
    gradientaccumulationsteps=2,
    learningrate=1e-5,

    warmupsteps=500,
    maxsteps=4000,

    fp16=True,
    evaluationstrategy="steps",
    evalsteps=500,

    savesteps=1000,
    loggingsteps=25,

    generationmaxlength=225,

    predictwithgenerate=True,

    reportto=["tensorboard"],
)

trainer = Seq2SeqTrainer(
    args=trainingargs,

    model=model,
    traindataset=dataset,
    datacollator=datacollator,
    tokenizer=processor.featureextractor,

)

trainer.train()
model.savepretrained("./whisper-finetuned-final")
processor.savepretrained("./whisper-finetuned-final")

import numpy as np
import sounddevice as sd
import whisper
import queue
import threading

class RealTimeTranscriber:
    def init(self, modelname: str = "base", samplerate: int = 16000, blockduration: float = 3.0):
        self.model = whisper.loadmodel(modelname)
        self.samplerate = samplerate
        self.blockduration = blockduration
        self.audioqueue = queue.Queue()

        self.isrunning = False
        self.buffer = np.array([], dtype=np.float32)

    def audiocallback(self, indata, frames, timeinfo, status):
        if status:
            print(f"Audio status: {status}")
        self.audioqueue.put(indata.copy().flatten())


    def processaudio(self):
        while self.isrunning:

            try:
                audiochunk = self.audioqueue.get(timeout=1.0)

                self.buffer = np.concatenate([self.buffer, audiochunk])

                # Process when buffer reaches desired duration
                minsamples = int(self.samplerate  self.blockduration)

                if len(self.buffer) >= minsamples:
                    audiodata = self.buffer.copy()

                    self.buffer = np.array([], dtype=np.float32)

                    # Normalize audio
                    audiodata = audiodata.astype(np.float32)

                    if np.max(np.abs(audiodata)) > 0:
                        audiodata = audiodata / np.max(np.abs(audiodata))


                    result = self.model.transcribe(
                        audiodata,
                        fp16=False,
                        language="en",
                        nospeechthreshold=0.5
                    )

                    text = result["text"].strip()
                    if text:
                        print(f">> {text}")

            except queue.Empty:
                continue

    def start(self):
        self.isrunning = True

        processthread = threading.Thread(target=self.processaudio, daemon=True)

        processthread.start()

        print("Listening... Press Ctrl+C to stop.")
        with sd.InputStream(
            samplerate=self.samplerate,

            channels=1,
            dtype="float32",
            callback=self.audiocallback,
            blocksize=int(self.samplerate  0.5)

        ):
            try:
                while True:
                    sd.sleep(100)
            except KeyboardInterrupt:
                self.isrunning = False
                print("\nStopped.")

Usage
transcriber = RealTimeTranscriber(modelname="base", blockduration=3.0)
transcriber.start()

from fasterwhisper import WhisperModel

Load model with CTranslate2 optimization
model = WhisperModel(
    "large-v3",
    device="cuda",
    computetype="float16",     # Options: float16, int8float16, int8

    cputhreads=4,
    numworkers=2

)

Transcribe with faster-whisper
segments, info = model.transcribe(
    "audiosample.mp3",
    beamsize=5,

    language="en",
    vadfilter=True,              # Voice Activity Detection filtering
    vadparameters=dict(

        minsilencedurationms=500,
        speechpadms=400
    ),
    wordtimestamps=True

)

print(f"Detected language: {info.language} (probability: {info.languageprobability:.2f})")
print(f"Duration: {info.duration:.2f}s")

for segment in segments:
    print(f"[{segment.start:.2f}s -> {segment.end:.2f}s] {segment.text}")
    if segment.words:
        for word in segment.words:
            print(f"    {word.word} ({word.start:.2f}s - {word.end:.2f}s) p={word.probability:.2f}")

import time

def benchmarkmodel(audiopath: str):
    # Standard Whisper
    import whisper
    modelstd = whisper.loadmodel("large-v3")
    start = time.time()
    resultstd = modelstd.transcribe(audiopath)

    timestd = time.time() - start

    # Faster-Whisper
    from fasterwhisper import WhisperModel

    modelfast = WhisperModel("large-v3", device="cuda", computetype="float16")

    start = time.time()
    segments,  = modelfast.transcribe(audiopath)
     = list(segments)  # Consume generator

    timefast = time.time() - start

    print(f"Standard Whisper: {timestd:.2f}s")

    print(f"Faster-Whisper:   {timefast:.2f}s")
    print(f"Speedup:          {timestd / timefast:.1f}x")

benchmarkmodel("testaudio.mp3")

from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks
from fastapi.responses import JSONResponse
from fasterwhisper import WhisperModel

import tempfile
import os
import uuid
from typing import Optional
from pydantic import BaseModel

app = FastAPI(title="Whisper Transcription API", version="1.0.0")

Global model instance
model = WhisperModel("medium", device="cuda", computetype="float16")

In-memory job storage (use Redis in production)
jobs: dict = {}

class TranscriptionResponse(BaseModel):
    text: str
    language: str
    duration: float
    segments: list

class JobStatus(BaseModel):
    jobid: str

    status: str
    result: Optional[TranscriptionResponse] = None

@app.post("/transcribe", responsemodel=TranscriptionResponse)
async def transcribeaudio(

    file: UploadFile = File(...),
    language: Optional[str] = None,
    task: str = "transcribe"
):
    if not file.filename.endswith((".mp3", ".wav", ".m4a", ".flac", ".ogg", ".webm")):
        raise HTTPException(statuscode=400, detail="Unsupported audio format")

    with tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(file.filename)[1]) as tmp:
        content = await file.read()
        tmp.write(content)
        tmppath = tmp.name


    try:
        segmentsgen, info = model.transcribe(
            tmppath,

            language=language,
            task=task,
            beamsize=5,
            vadfilter=True

        )

        segmentslist = []
        fulltextparts = []
        for seg in segmentsgen:

            segmentslist.append({
                "start": round(seg.start, 2),
                "end": round(seg.end, 2),
                "text": seg.text.strip()
            })
            fulltextparts.append(seg.text.strip())

        return TranscriptionResponse(
            text=" ".join(fulltextparts),
            language=info.language,
            duration=round(info.duration, 2),
            segments=segmentslist

        )
    finally:
        os.unlink(tmppath)

def processtranscriptionjob(jobid: str, filepath: str, language: Optional[str]):
    try:
        segmentsgen, info = model.transcribe(filepath, language=language, vadfilter=True)

        segmentslist = []
        fulltext = []

        for seg in segmentsgen:
            segmentslist.append({"start": seg.start, "end": seg.end, "text": seg.text.strip()})

            fulltext.append(seg.text.strip())

        jobs[jobid] = {

            "status": "completed",
            "result": {
                "text": " ".join(fulltext),
                "language": info.language,
                "duration": info.duration,
                "segments": segmentslist

            }
        }
    except Exception as e:
        jobs[jobid] = {"status": "failed", "error": str(e)}
    finally:
        os.unlink(filepath)


@app.post("/transcribe/async")
async def transcribeasync(
    backgroundtasks: BackgroundTasks,

    file: UploadFile = File(...),
    language: Optional[str] = None
):
    jobid = str(uuid.uuid4())
    with tempfile.NamedTemporaryFile(delete=False, suffix=os.path.splitext(file.filename)[1]) as tmp:
        content = await file.read()
        tmp.write(content)
        tmppath = tmp.name


    jobs[jobid] = {"status": "processing"}
    backgroundtasks.addtask(processtranscriptionjob, jobid, tmppath, language)
    return {"jobid": jobid, "status": "processing"}

@app.get("/transcribe/status/{jobid}")

async def getjobstatus(jobid: str):
    if jobid not in jobs:

        raise HTTPException(statuscode=404, detail="Job not found")
    return jobs[jobid]


Run with: uvicorn whisperapi:app --host 0.0.0.0 --port 8000

from fasterwhisper import WhisperModel
from pathlib import Path

def formattimestampsrt(seconds: float) -> str:

    hours = int(seconds // 3600)
    minutes = int((seconds % 3600) // 60)
    secs = int(seconds % 60)
    millis = int((seconds % 1)  1000)
    return f"{hours:02d}:{minutes:02d}:{secs:02d},{millis:03d}"

def formattimestampvtt(seconds: float) -> str:

    hours = int(seconds // 3600)
    minutes = int((seconds % 3600) // 60)
    secs = int(seconds % 60)
    millis = int((seconds % 1) * 1000)
    return f"{hours:02d}:{minutes:02d}:{secs:02d}.{millis:03d}"

def generatesrt(audiopath: str, outputpath: str, maxcharsperline: int = 42):

    model = WhisperModel("medium", device="cuda", computetype="float16")
    segments, info = model.transcribe(audiopath, wordtimestamps=True, vadfilter=True)


    srtcontent = []
    index = 1

    for segment in segments:
        text = segment.text.strip()
        # Split long lines
        if len(text) > maxcharsperline:

            mid = len(text) // 2
            spacepos = text.rfind(" ", 0, mid)
            if spacepos != -1:

                text = text[:spacepos] + "\n" + text[spacepos + 1:]


        start = formattimestampsrt(segment.start)

        end = formattimestampsrt(segment.end)

        srtcontent.append(f"{index}\n{start} --> {end}\n{text}\n")
        index += 1

    with open(outputpath, "w", encoding="utf-8") as f:

        f.write("\n".join(srtcontent))
    print(f"SRT saved: {outputpath} ({index - 1} subtitles)")


def generatevtt(audiopath: str, outputpath: str):
    model = WhisperModel("medium", device="cuda", computetype="float16")

    segments, info = model.transcribe(audiopath, wordtimestamps=True, vadfilter=True)

    vttlines = ["WEBVTT", ""]

    for segment in segments:
        start = formattimestampvtt(segment.start)

        end = formattimestampvtt(segment.end)

        vttlines.append(f"{start} --> {end}")
        vttlines.append(segment.text.strip())

        vttlines.append("")

    with open(outputpath, "w", encoding="utf-8") as f:

        f.write("\n".join(vttlines))
    print(f"VTT saved: {outputpath}")


Generate both formats
generatesrt("lecture.mp3", "lecture.srt")
generatevtt("lecture.mp3", "lecture.vtt")

   ffmpeg -i input.mp4 -ar 16000 -ac 1 -c:a pcms16le output.wav

   

Conclusion

OpenAI Whisper is a powerful and versatile speech recognition system. By combining it with faster-whisper for optimization, FastAPI for serving, and proper fine-tuning for domain adaptation, you can build production-grade transcription systems that handle diverse languages and audio conditions. The key is choosing the right model size for your latency and accuracy requirements, and leveraging VAD filtering and proper audio preprocessing to maximize quality.