ColPali Tutorial: Visual Document Retrieval with Vision-Language Models
Visual document retrieval is one of the biggest challenges in AI today. Imagine having thousands of PDF documents containing tables, charts, diagrams, and complex layouts. Traditional methods using OCR and text extraction often fail to capture crucial visual context. ColPali emerges as a groundbreaking solution that uses vision-language models to perform retrieval directly from document images without requiring OCR.
In this tutorial, we will learn how to use ColPali to build a document retrieval system that can understand visual document content directly.
What Is ColPali?
ColPali is a retrieval model that combines the ColBERT (Contextualized Late Interaction over BERT) architecture with PaliGemma (a vision-language model from Google). Instead of extracting text from documents and then performing retrieval on that text, ColPali directly processes document page images and produces multi-vector embeddings that represent both visual and textual content simultaneously.
Key advantages of ColPali:
- No OCR needed: Eliminates complex and error-prone OCR pipelines
- Layout-aware: Captures information from tables, charts, and diagrams
- Multi-vector retrieval: Uses late interaction like ColBERT for more precise matching
- End-to-end: A single model for the entire retrieval pipeline
Installation
System Requirements
- Python 3.10 or newer
- GPU with at least 8GB VRAM (16GB+ recommended)
- CUDA 11.8 or newer
Library Installation
pip install colpali-engine torch pillow pdf2image transformers
For PDF to image conversion, you also need poppler:
# Ubuntu/Debian
sudo apt-get install poppler-utils
macOS
brew install poppler
Windows (via conda)
conda install -c conda-forge poppler
Optional Installation
If you want to use Qdrant as a vector store:
pip install qdrant-client
Basic Usage
Loading the ColPali Model
import torch
from colpaliengine.models import ColPali, ColPaliProcessor
modelname = "vidore/colpali-v1.3"
model = ColPali.frompretrained(
modelname,
torchdtype=torch.bfloat16,
devicemap="cuda"
)
processor = ColPaliProcessor.frompretrained(modelname)
Indexing Documents
The first step is converting document pages into embeddings:
from PIL import Image
from pdf2image import convertfrompath
def indexpdf(pdfpath, model, processor):
"""Convert PDF to per-page embeddings."""
pages = convertfrompath(pdfpath, dpi=144)
allembeddings = []
for i, page in enumerate(pages):
inputs = processor.processimages([page]).to(model.device)
with torch.nograd():
embeddings = model(inputs)
allembeddings.append({
"pagenum": i + 1,
"embeddings": embeddings.cpu(),
"image": page
})
print(f"Indexed page {i + 1}/{len(pages)}")
return allembeddings
Example usage
docembeddings = indexpdf("financialreport.pdf", model, processor)
print(f"Total pages indexed: {len(docembeddings)}")
Performing Queries
After documents are indexed, we can perform searches:
def searchdocuments(query, docembeddings, model, processor, topk=3):
"""Search for the most relevant document pages for a query."""
inputs = processor.processqueries([query]).to(model.device)
with torch.nograd():
queryembeddings = model(inputs)
scores = []
for doc in docembeddings:
score = processor.scoremultivector(
queryembeddings,
doc["embeddings"].to(model.device)
)
scores.append({
"pagenum": doc["pagenum"],
"score": score.item(),
"image": doc["image"]
})
scores.sort(key=lambda x: x["score"], reverse=True)
return scores[:topk]
Example search
results = searchdocuments(
"What is the total revenue for Q3 2025?",
docembeddings,
model,
processor
)
for r in results:
print(f"Page {r['pagenum']}: Score {r['score']:.4f}")
Visualizing Results
import matplotlib.pyplot as plt
def visualizeresults(query, results, maxdisplay=3):
"""Display search results as images."""
fig, axes = plt.subplots(1, min(len(results), maxdisplay), figsize=(18, 8))
if maxdisplay == 1:
axes = [axes]
fig.suptitle(f'Query: "{query}"', fontsize=14, fontweight="bold")
for idx, (ax, result) in enumerate(zip(axes, results[:maxdisplay])):
ax.imshow(result["image"])
ax.settitle(f"Page {result['pagenum']}\nScore: {result['score']:.4f}")
ax.axis("off")
plt.tightlayout()
plt.savefig("searchresults.png", dpi=150, bboxinches="tight")
plt.show()
visualizeresults("Total revenue Q3 2025", results)
Advanced Usage
Multi-Document Retrieval System
For real-world use cases, you typically need to index many documents at once:
import os
import json
from pathlib import Path
class ColPaliRetriever:
def init(self, modelname="vidore/colpali-v1.3"):
self.model = ColPali.frompretrained(
modelname,
torchdtype=torch.bfloat16,
devicemap="cuda"
)
self.processor = ColPaliProcessor.frompretrained(modelname)
self.index = []
def adddocument(self, pdfpath, docid=None):
"""Add a document to the index."""
if docid is None:
docid = Path(pdfpath).stem
pages = convertfrompath(pdfpath, dpi=144)
for i, page in enumerate(pages):
inputs = self.processor.processimages([page]).to(self.model.device)
with torch.nograd():
embeddings = self.model(*inputs)
self.index.append({
"docid": docid,
"pagenum": i + 1,
"embeddings": embeddings.cpu(),
"source": str(pdfpath)
})
print(f"Added '{docid}': {len(pages)} pages")
def adddirectory(self, directory, pattern=".pdf"):
"""Index all PDFs in a directory."""
pdffiles = sorted(Path(directory).glob(pattern))
for pdf in pdffiles:
self.adddocument(str(pdf))
print(f"Total indexed: {len(self.index)} pages from {len(pdffiles)} documents")
def search(self, query, topk=5):
"""Search for the most relevant pages across the entire index."""
inputs = self.processor.processqueries([query]).to(self.model.device)
with torch.nograd():
queryembeddings = self.model(inputs)
results = []
for entry in self.index:
score = self.processor.scoremultivector(
queryembeddings,
entry["embeddings"].to(self.model.device)
)
results.append({
"docid": entry["docid"],
"pagenum": entry["pagenum"],
"score": score.item(),
"source": entry["source"]
})
results.sort(key=lambda x: x["score"], reverse=True)
return results[:topk]
Usage
retriever = ColPaliRetriever()
retriever.adddirectory("./documents/")
results = retriever.search("annual sales growth chart")
for r in results:
print(f"[{r['docid']}] Page {r['pagenum']}: {r['score']:.4f}")
Integration with Qdrant for Production
For production deployment, store embeddings in a vector database:
from qdrantclient import QdrantClient
from qdrantclient.models import (
Distance, VectorParams, PointStruct,
NamedVector, models
)
import numpy as np
import uuid
class ColPaliQdrantRetriever:
def init(self, modelname="vidore/colpali-v1.3", qdranturl="localhost", qdrantport=6333):
self.model = ColPali.frompretrained(
modelname,
torchdtype=torch.bfloat16,
devicemap="cuda"
)
self.processor = ColPaliProcessor.frompretrained(modelname)
self.client = QdrantClient(host=qdranturl, port=qdrantport)
self.collectionname = "colpalidocuments"
self.createcollection()
def createcollection(self):
"""Create Qdrant collection if it doesn't exist."""
collections = [c.name for c in self.client.getcollections().collections]
if self.collectionname not in collections:
self.client.createcollection(
collectionname=self.collectionname,
vectorsconfig={
"colpali": VectorParams(
size=128,
distance=Distance.COSINE,
multivectorconfig=models.MultiVectorConfig(
comparator=models.MultiVectorComparator.MAXSIM
)
)
}
)
def indexdocument(self, pdfpath, metadata=None):
"""Index a document into Qdrant."""
pages = convertfrompath(pdfpath, dpi=144)
points = []
for i, page in enumerate(pages):
inputs = self.processor.processimages([page]).to(self.model.device)
with torch.nograd():
embeddings = self.model(inputs)
emblist = embeddings.cpu().float().numpy().tolist()[0]
point = PointStruct(
id=str(uuid.uuid4()),
vector={"colpali": emblist},
payload={
"source": str(pdfpath),
"pagenum": i + 1,
"totalpages": len(pages),
(metadata or {})
}
)
points.append(point)
self.client.upsert(
collectionname=self.collectionname,
points=points
)
print(f"Indexed {len(points)} pages from {pdfpath}")
def search(self, query, topk=5):
"""Multi-vector search in Qdrant."""
inputs = self.processor.processqueries([query]).to(self.model.device)
with torch.nograd():
queryembeddings = self.model(inputs)
querylist = queryembeddings.cpu().float().numpy().tolist()[0]
results = self.client.querypoints(
collectionname=self.collectionname,
query=querylist,
using="colpali",
limit=topk
)
return [
{
"score": r.score,
"source": r.payload["source"],
"pagenum": r.payload["pagenum"]
}
for r in results.points
]
Usage
retriever = ColPaliQdrantRetriever()
retriever.indexdocument("annualreport.pdf", metadata={"type": "financial"})
results = retriever.search("revenue breakdown by region")
Batch Processing for Large Datasets
To efficiently index thousands of documents:
from torch.utils.data import DataLoader, Dataset
class PDFPageDataset(Dataset):
def init(self, pdfpaths, dpi=144):
self.pages = []
for pdfpath in pdfpaths:
images = convertfrompath(pdfpath, dpi=dpi)
for i, img in enumerate(images):
self.pages.append({
"image": img,
"source": str(pdfpath),
"pagenum": i + 1
})
def len(self):
return len(self.pages)
def getitem(self, idx):
return self.pages[idx]
def batchindex(pdfpaths, model, processor, batchsize=4):
"""Index documents in batches for GPU efficiency."""
dataset = PDFPageDataset(pdfpaths)
allresults = []
for i in range(0, len(dataset), batchsize):
batch = [dataset[j] for j in range(i, min(i + batchsize, len(dataset)))]
images = [item["image"] for item in batch]
inputs = processor.processimages(images).to(model.device)
with torch.nograd():
embeddings = model(*inputs)
for j, item in enumerate(batch):
allresults.append({
"source": item["source"],
"pagenum": item["pagenum"],
"embeddings": embeddings[j:j+1].cpu()
})
print(f"Processed {min(i + batchsize, len(dataset))}/{len(dataset)} pages")
return allresults
ColPali + RAG Pipeline
Combine ColPali with an LLM for question answering:
import base64
from io import BytesIO
from openai import OpenAI
class VisualRAG:
def init(self, retriever):
self.retriever = retriever
self.llmclient = OpenAI()
def imagetobase64(self, image):
"""Convert PIL Image to base64."""
buffer = BytesIO()
image.save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode()
def answer(self, question, topk=3):
"""Answer a question based on retrieved documents."""
results = self.retriever.search(question, topk=topk)
imagecontents = []
for r in results:
pages = convertfrompath(r["source"], dpi=144,
firstpage=r["pagenum"],
lastpage=r["pagenum"])
if pages:
b64 = self.imagetobase64(pages[0])
imagecontents.append({
"type": "imageurl",
"imageurl": {
"url": f"data:image/png;base64,{b64}",
"detail": "high"
}
})
messages = [
{
"role": "user",
"content": [
{"type": "text", "text": f"Based on the following document pages, answer this question: {question}"},
imagecontents
]
}
]
response = self.llmclient.chat.completions.create(
model="gpt-4o",
messages=messages,
maxtokens=1024
)
return {
"answer": response.choices[0].message.content,
"sources": [
{"source": r["source"], "page": r["pagenum"], "score": r["score"]}
for r in results
]
}
Usage
rag = VisualRAG(retriever)
result = rag.answer("What is the company's total assets in 2025?")
print(result["answer"])
Best Practices
1. Image Resolution Optimization
DPI resolution significantly affects retrieval quality. Use 144 DPI as a good default:
# Too low - text details are lost
pageslow = convertfrompath(pdfpath, dpi=72)
Optimal - balance between quality and speed
pagesoptimal = convertfrompath(pdfpath, dpi=144)
Too high - slow without significant improvement
pageshigh = convertfrompath(pdfpath, dpi=300)
2. GPU Memory Management
ColPali uses significant GPU memory. Some tips:
# Use bfloat16 to reduce memory usage
model = ColPali.frompretrained(
modelname,
torchdtype=torch.bfloat16,
devicemap="cuda"
)
Clear cache after batch processing
torch.cuda.emptycache()
Enable gradient checkpointing if memory is limited
model.gradientcheckpointingenable()
3. Document Preprocessing
Not all pages need to be indexed. Filter irrelevant pages:
def shouldindexpage(image, mintextratio=0.05):
"""Simple heuristic to determine if a page should be indexed."""
import numpy as np
img
array = np.array(image.convert("L"))
nonwhite = np.sum(imgarray < 240)
total = imgarray.size
return (nonwhite / total) > mintextratio
Filter blank or nearly blank pages
pages = convertfrompath(pdfpath, dpi=144)
pagestoindex = [p for p in pages if shouldindexpage(p)]
4. Embedding Caching
Save embeddings to disk to avoid recomputation:
import pickle
from pathlib import Path
def getorcreateembeddings(pdfpath, model, processor, cachedir="./cache"):
"""Load embeddings from cache or create new ones."""
cachepath = Path(cachedir) / f"{Path(pdfpath).stem}.pkl"
cachepath.parent.mkdir(existok=True)
if cachepath.exists():
with open(cachepath, "rb") as f:
return pickle.load(f)
embeddings = indexpdf(pdfpath, model, processor)
with open(cachepath, "wb") as f:
pickle.dump(embeddings, f)
return embeddings
5. Retrieval Quality Evaluation
Measure your retrieval system's performance:
def evaluateretrieval(retriever, testqueries, groundtruth):
"""Evaluate retrieval accuracy."""
metrics = {"mrr": 0, "recall
at5": 0, "recallat10": 0}
n = len(test
queries)
for query, expected in zip(testqueries, groundtruth):
results = retriever.search(query, topk=10)
retrievedpages = [(r["docid"], r["pagenum"]) for r in results]
for rank, page in enumerate(retrievedpages):
if page in expected:
metrics["mrr"] += 1 / (rank + 1)
break
top5 = set(retrievedpages[:5])
top10 = set(retrievedpages[:10])
metrics["recallat5"] += len(top5 & set(expected)) / len(expected)
metrics["recallat_10"] += len(top10 & set(expected)) / len(expected)
return {k: v / n for k, v in metrics.items()}
Comparison with Traditional Methods
| Aspect | OCR + Text Retrieval | ColPali |
|--------|---------------------|---------|
| Setup | Complex (OCR pipeline) | Simple (1 model) |
| Tables & Charts | Often fails | Good |
| Indexing Speed | Fast | Slower |
| Accuracy | Depends on OCR | Consistently high |
| GPU Required | Not mandatory | Yes |
| Multi-language | Requires OCR config | Built-in |
Conclusion
ColPali brings a paradigm shift in how we approach document retrieval. By eliminating the need for OCR and directly processing document visuals, ColPali captures information that traditional methods often miss, such as tables, charts, and complex layouts.
Key takeaways from this tutorial:
For production projects, make sure to use a vector database like Qdrant or Milvus to store embeddings, implement proper caching, and regularly evaluate your retrieval system's quality.