bitsandbytes Tutorial: 4-bit and 8-bit Quantization for Large Language Models

Running Large Language Models (LLMs) like Llama, Mistral, or Falcon requires GPUs with substantial VRAM. A 7-billion parameter model alone can consume 14 GB of VRAM in float16 precision. What if you only have a consumer-grade GPU with 8 GB of VRAM? This is where bitsandbytes comes in.

bitsandbytes is a Python library that enables you to run deep learning models in 4-bit or 8-bit precision, drastically reducing memory requirements without significantly sacrificing output quality. This library serves as the foundation for QLoRA (Quantized Low-Rank Adaptation), which enables fine-tuning billion-parameter models on consumer GPUs.

In this tutorial, you will learn how to use bitsandbytes to:

• Load LLM models in 8-bit and 4-bit precision
• Configure quantization parameters for optimal results
• Perform fine-tuning with QLoRA
• Optimize GPU memory usage

Installation

Prerequisites

bitsandbytes requires an NVIDIA GPU with CUDA support. Make sure you have:

• Python 3.8 or newer
• PyTorch 2.0 or newer
• CUDA Toolkit 11.8 or 12.x
• NVIDIA GPU (compute capability 7.0+, i.e., Volta or newer)

Installing Libraries

pip install bitsandbytes
pip install transformers accelerate
pip install peft datasets  # for QLoRA fine-tuning

To verify the installation:

import bitsandbytes as bnb
import torch

print(f"bitsandbytes version: {bnb.version}")
print(f"PyTorch version: {torch.version}")
print(f"CUDA available: {torch.cuda.isavailable()}")

print(f"GPU: {torch.cuda.getdevicename(0)}")

from transformers import AutoModelForCausalLM, AutoTokenizer

modelname = "mistralai/Mistral-7B-Instruct-v0.3"

tokenizer = AutoTokenizer.frompretrained(modelname)

model = AutoModelForCausalLM.frompretrained(

    modelname,
    loadin8bit=True,
    devicemap="auto"

)

prompt = "Explain what machine learning is in 3 sentences:"
inputs = tokenizer(prompt, returntensors="pt").to(model.device)

outputs = model.generate(
    inputs,

    maxnewtokens=150,
    temperature=0.7,
    dosample=True

)

response = tokenizer.decode(outputs[0], skipspecialtokens=True)

print(response)

import torch

def printgpumemory():
    if torch.cuda.isavailable():

        allocated = torch.cuda.memoryallocated() / 10243
        reserved = torch.cuda.memoryreserved() / 10243

        print(f"GPU Memory Allocated: {allocated:.2f} GB")
        print(f"GPU Memory Reserved: {reserved:.2f} GB")

printgpumemory()

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

quantizationconfig = BitsAndBytesConfig(
    loadin4bit=True,
    bnb4bitquanttype="nf4",

    bnb4bitcomputedtype=torch.bfloat16,
    bnb4bitusedoublequant=True
)

modelname = "mistralai/Mistral-7B-Instruct-v0.3"


tokenizer = AutoTokenizer.frompretrained(modelname)


model = AutoModelForCausalLM.frompretrained(
    modelname,

    quantizationconfig=quantizationconfig,

    devicemap="auto"
)

printgpumemory()

# Comparing NF4 vs FP4
confignf4 = BitsAndBytesConfig(
    loadin4bit=True,
    bnb4bitquanttype="nf4",

    bnb4bitcomputedtype=torch.bfloat16
)

configfp4 = BitsAndBytesConfig(

    loadin4bit=True,

    bnb4bitquanttype="fp4",
    bnb4bitcomputedtype=torch.bfloat16

)

# With double quant: ~3.5 GB for 7B model
configdouble = BitsAndBytesConfig(

    loadin4bit=True,

    bnb4bitquanttype="nf4",
    bnb4bitcomputedtype=torch.bfloat16,

    bnb4bitusedoublequant=True

)

Without double quant: ~3.9 GB for 7B model
configsingle = BitsAndBytesConfig(
    loadin4bit=True,
    bnb4bitquanttype="nf4",

    bnb4bitcomputedtype=torch.bfloat16,
    bnb4bitusedoublequant=False
)

from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import torch

quantizationconfig = BitsAndBytesConfig(

    loadin4bit=True,

    bnb4bitquanttype="nf4",
    bnb4bitcomputedtype=torch.bfloat16,

    bnb4bitusedoublequant=True

)

modelname = "mistralai/Mistral-7B-Instruct-v0.3"
tokenizer = AutoTokenizer.frompretrained(modelname)
model = AutoModelForCausalLM.frompretrained(

    modelname,
    quantizationconfig=quantizationconfig,
    devicemap="auto"

)

def generateresponse(prompt, maxtokens=256):

    messages = [
        {"role": "user", "content": prompt}
    ]

    inputids = tokenizer.applychattemplate(
        messages,
        returntensors="pt",

        addgenerationprompt=True

    ).to(model.device)

    outputs = model.generate(
        inputids,
        maxnewtokens=maxtokens,

        temperature=0.7,
        topp=0.9,
        dosample=True,

        repetitionpenalty=1.1
    )

    response = tokenizer.decode(
        outputs[0][inputids.shape[1]:],

        skipspecialtokens=True

    )
    return response

print(generateresponse("What are the advantages of using Python for data science?"))
print(generateresponse("Write a Python function to calculate fibonacci:"))

def batchgenerate(prompts, maxtokens=256):
    tokenizer.padtoken = tokenizer.eostoken


    inputs = tokenizer(
        prompts,
        returntensors="pt",
        padding=True,
        truncation=True,
        maxlength=512

    ).to(model.device)

    outputs = model.generate(
        inputs,
        maxnewtokens=maxtokens,
        temperature=0.7,
        dosample=True

    )

    responses = tokenizer.batchdecode(outputs, skipspecialtokens=True)
    return responses

prompts = [
    "Explain the concept of OOP in one paragraph:",
    "What is a REST API?",
    "Name 3 popular databases:"
]

results = batchgenerate(prompts)

for prompt, result in zip(prompts, results):
    print(f"Q: {prompt}")
    print(f"A: {result}\n")

from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    TrainingArguments
)
from peft import LoraConfig, getpeftmodel, preparemodelforkbittraining

from datasets import loaddataset
from trl import SFTTrainer
import torch

1. Quantization configuration
bnbconfig = BitsAndBytesConfig(

    loadin4bit=True,

    bnb4bitquanttype="nf4",
    bnb4bitcomputedtype=torch.bfloat16,

    bnb4bitusedoublequant=True

)

2. Load model and tokenizer
modelname = "mistralai/Mistral-7B-Instruct-v0.3"
tokenizer = AutoTokenizer.frompretrained(modelname)
tokenizer.padtoken = tokenizer.eostoken

model = AutoModelForCausalLM.frompretrained(

    modelname,
    quantizationconfig=bnbconfig,
    devicemap="auto",

    torchdtype=torch.bfloat16
)

3. Prepare model for k-bit training
model = preparemodelforkbittraining(model)

4. LoRA configuration
loraconfig = LoraConfig(

    r=16,
    loraalpha=32,
    targetmodules=[

        "qproj", "kproj", "vproj", "oproj",

        "gateproj", "upproj", "downproj"
    ],
    loradropout=0.05,

    bias="none",
    tasktype="CAUSALLM"

)

5. Apply LoRA to model
model = getpeftmodel(model, loraconfig)
model.printtrainableparameters()

6. Load dataset
dataset = loaddataset("timdettmers/openassistant-guanaco", split="train[:5000]")


7. Format data
def formatinstruction(sample):
    return f"""### Human: {sample['text'].split('### Human:')[1].split('### Assistant:')[0].strip()}

Assistant: {sample['text'].split('### Assistant:')[1].strip()}"""

8. Training configuration
trainingargs = TrainingArguments(

    outputdir="./qlora-output",
    numtrainepochs=1,
    perdevicetrainbatchsize=4,
    gradientaccumulationsteps=4,
    learningrate=2e-4,

    weightdecay=0.001,
    warmupratio=0.03,

    lrschedulertype="cosine",

    loggingsteps=10,
    savestrategy="steps",

    savesteps=100,
    fp16=False,
    bf16=True,
    optim="pagedadamw32bit",
    gradientcheckpointing=True,

    maxgradnorm=0.3,

    reportto="none"
)

9. Initialize trainer
trainer = SFTTrainer(
    model=model,
    traindataset=dataset,

    tokenizer=tokenizer,
    args=trainingargs,
    maxseqlength=512,
)

10. Start training
trainer.train()

11. Save adapter
trainer.savemodel("./qlora-adapter")

# r=8: ~0.1% trainable params, fast, for simple tasks
r=16: ~0.2% trainable params, good balance
r=64: ~0.8% trainable params, for complex tasks

import bitsandbytes as bnb

optimizer = bnb.optim.PagedAdamW32bit(
    model.parameters(),
    lr=2e-4,
    weightdecay=0.001
)

Or use 8-bit Adam for even more memory savings
optimizer8bit = bnb.optim.Adam8bit(

    model.parameters(),
    lr=2e-4,
    weightdecay=0.001
)

Paged optimizers work by leveraging CUDA unified memory for automatic page fault handling, similar to virtual memory in operating systems.

Saving and Loading Quantized Models

Saving QLoRA Adapters

# Save only the adapter (small size, ~50-200 MB)
model.savepretrained("./my-qlora-adapter")

tokenizer.savepretrained("./my-qlora-adapter")

Loading Adapters Back

from peft import PeftModel

Load base model in 4-bit
basemodel = AutoModelForCausalLM.frompretrained(
    "mistralai/Mistral-7B-Instruct-v0.3",
    quantizationconfig=bnbconfig,
    devicemap="auto"

)

Apply adapter
model = PeftModel.frompretrained(basemodel, "./my-qlora-adapter")

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

Load base model in full precision for merging
basemodel = AutoModelForCausalLM.frompretrained(

    "mistralai/Mistral-7B-Instruct-v0.3",
    torchdtype=torch.float16,
    devicemap="auto"

)

model = PeftModel.frompretrained(basemodel, "./my-qlora-adapter")

mergedmodel = model.mergeandunload()

mergedmodel.savepretrained("./merged-model")
tokenizer.savepretrained("./merged-model")

import bitsandbytes as bnb

Drop-in replacement for torch.optim.Adam
optimizer = bnb.optim.Adam8bit(
    model.parameters(),
    lr=1e-4,
    betas=(0.9, 0.999)
)

Also available:
bnb.optim.AdamW8bit - AdamW with weight decay
bnb.optim.SGD8bit - 8-bit SGD
bnb.optim.Adagrad8bit - 8-bit Adagrad
bnb.optim.LAMB8bit - 8-bit LAMB (for large batches)

import bitsandbytes as bnb
import torch.nn as nn

class EfficientModel(nn.Module):
    def init(self, inputdim, hiddendim, outputdim):
        super().init()
        self.layer1 = bnb.nn.Linear8bitLt(inputdim, hiddendim, hasfp16weights=False)
        self.activation = nn.GELU()
        self.layer2 = bnb.nn.Linear8bitLt(hiddendim, outputdim, hasfp16weights=False)

    def forward(self, x):
        x = self.layer1(x)
        x = self.activation(x)
        x = self.layer2(x)
        return x

Best Practices

1. Choose the Right Quantization Method

| Scenario | Recommendation |

|----------|---------------|

| Inference only, adequate GPU | 8-bit (LLM.int8()) |

| Inference only, limited GPU | 4-bit NF4 + double quant |

| Fine-tuning | 4-bit NF4 + QLoRA |

| Production, need speed | GPTQ or AWQ |

| Research, need flexibility | bitsandbytes 4-bit |

2. Use bfloat16 as Compute Dtype

# Recommended for Ampere+ GPUs
config = BitsAndBytesConfig(
    loadin4bit=True,
    bnb4bitcomputedtype=torch.bfloat16  # not float16

)

model.gradientcheckpointingenable()

Or through TrainingArguments
trainingargs = TrainingArguments(
    gradientcheckpointing=True,

    # ...
)

def printmemorystats():
    print(f"Allocated: {torch.cuda.memoryallocated() / 10243:.2f} GB")

    print(f"Reserved: {torch.cuda.memoryreserved() / 10243:.2f} GB")

    print(f"Max Allocated: {torch.cuda.maxmemoryallocated() / 10243:.2f} GB")


Call before and after heavy operations
printmemorystats()

# Reduce batch size
trainingargs = TrainingArguments(perdevicetrainbatchsize=1)

Enable gradient accumulation
trainingargs = TrainingArguments(

    perdevicetrainbatchsize=1,

    gradientaccumulation_steps=16
)

Installation fails on Linux:

# Make sure CUDA toolkit is installed
nvcc --version

Install specific version
pip install bitsandbytes==0.43.0

Model produces poor output after quantization:

• For inference: Use 4-bit NF4 with double quantization for maximum memory savings
• For fine-tuning: Combine with PEFT/LoRA for QLoRA, enabling training on consumer GPUs
• 8-bit optimizers: Use Adam8bit or PagedAdamW32bit to reduce optimizer state memory
• Choose the right compute dtype**: bfloat16 for Ampere+ GPUs, float16 for older GPUs

With bitsandbytes, the barrier to working with LLMs is lower than ever. You can run a 7B parameter model on a GPU with just 6 GB of VRAM, or fine-tune a 13B model on a 16 GB GPU. This opens up tremendous opportunities for experimentation, research, and more accessible AI model deployment.