Multi-Task Batch Processing

def multitaskprocess(imagepath, tasks):
    """Run multiple tasks on a single image."""
    image = loadimage(imagepath)

    results = {}

    for taskprompt in tasks:
        result = runflorence(image, taskprompt)
        results[taskprompt] = result[taskprompt]

    return results

Run multiple tasks on one image
tasks = ["", "", ""]
results = multitaskprocess("example.jpg", tasks)

for task, result in results.items():
    print(f"\n{task}: {result}")

Fine-tuning on Custom Dataset

Fine-tuning Florence-2 allows you to improve performance on specific domains.

Dataset Preparation

from datasets import Dataset
from PIL import Image
import json

def preparedataset(annotationsfile, imagesdir):

    """Prepare dataset for Florence-2 fine-tuning."""
    with open(annotationsfile, "r") as f:
        annotations = json.load(f)

    data = {
        "image": [],
        "prefix": [],
        "suffix": [],
    }

    for ann in annotations:
        imagepath = os.path.join(imagesdir, ann["imagefilename"])

        image = Image.open(imagepath).convert("RGB")

        # For object detection task
        if "objects" in ann:
            prefix = ""
            # Output format: label1x1>y1>x2>y2>label2...
            suffixparts = []

            for obj in ann["objects"]:
                bbox = obj["bbox"]  # [x1, y1, x2, y2] normalized
                label = obj["label"]
                x1 = int(bbox[0]  999)
                y1 = int(bbox[1]  999)

                x2 = int(bbox[2]  999)
                y2 = int(bbox[3]  999)

                suffixparts.append(
                    f"{label}{x1}>{y1}>{x2}>{y2}>"
                )
            suffix = "".join(suffixparts)


        # For captioning task
        elif "caption" in ann:
            prefix = ""
            suffix = ann["caption"]

        data["image"].append(image)
        data["prefix"].append(prefix)
        data["suffix"].append(suffix)

    return Dataset.fromdict(data)

Prepare datasets
traindataset = preparedataset("trainannotations.json", "trainimages/")
valdataset = preparedataset("valannotations.json", "valimages/")

print(f"Training dataset: {len(traindataset)} samples")

print(f"Validation dataset: {len(valdataset)} samples")

Configuration and Training

from transformers import TrainingArguments, Trainer
import torch

class Florence2Dataset(torch.utils.data.Dataset):
    """Custom dataset for Florence-2 fine-tuning."""

    def init(self, dataset, processor, maxlength=1024):

        self.dataset = dataset
        self.processor = processor
        self.maxlength = maxlength


    def len(self):
        return len(self.dataset)

    def getitem(self, idx):
        item = self.dataset[idx]
        image = item["image"]
        prefix = item["prefix"]
        suffix = item["suffix"]

        inputs = self.processor(
            text=prefix,
            images=image,
            returntensors="pt",
            maxlength=self.maxlength,
            padding="maxlength",

            truncation=True
        )

        labels = self.processor.tokenizer(
            suffix,
            returntensors="pt",
            maxlength=self.maxlength,
            padding="maxlength",

            truncation=True
        )

        return {
            "inputids": inputs["inputids"].squeeze(),

            "pixelvalues": inputs["pixelvalues"].squeeze(),

            "labels": labels["inputids"].squeeze(),
        }

Create dataset loaders
traintorchdataset = Florence2Dataset(traindataset, processor)

valtorchdataset = Florence2Dataset(valdataset, processor)

Training configuration
trainingargs = TrainingArguments(

    outputdir="./florence2-finetuned",
    numtrainepochs=10,
    perdevicetrainbatchsize=4,
    perdeviceevalbatchsize=4,
    gradientaccumulationsteps=4,
    learningrate=1e-5,

    weightdecay=0.01,
    warmupratio=0.1,

    lrschedulertype="cosine",

    loggingsteps=50,
    evalstrategy="steps",

    evalsteps=200,
    savestrategy="steps",

    savesteps=200,
    savetotallimit=3,
    loadbestmodelatend=True,
    metricforbestmodel="evalloss",
    fp16=True,
    dataloadernumworkers=4,
    removeunusedcolumns=False,
    reportto="tensorboard",

)

Initialize Trainer
trainer = Trainer(
    model=model,
    args=trainingargs,
    traindataset=traintorchdataset,

    evaldataset=valtorchdataset,
)

Start training
trainer.train()

Save the model
trainer.savemodel("./florence2-finetuned-final")

processor.savepretrained("./florence2-finetuned-final")
print("Model saved successfully!")

Fine-tuning with LoRA (Memory Efficient)

from peft import LoraConfig, getpeftmodel

LoRA configuration
loraconfig = LoraConfig(

    r=16,
    loraalpha=32,
    targetmodules=["qproj", "vproj", "kproj", "oproj"],

    loradropout=0.05,
    bias="none",
    tasktype="CAUSALLM",
)

Apply LoRA to the model
peftmodel = getpeftmodel(model, loraconfig)
peftmodel.printtrainableparameters()

Output: trainable params: 2,359,296 || all params: 234,567,890 || trainable%: 1.01%

Use peftmodel instead of model in Trainer
trainer = Trainer(
    model=peftmodel,

    args=trainingargs,
    traindataset=traintorchdataset,

    evaldataset=valtorchdataset,
)

trainer.train()
peftmodel.savepretrained("./florence2-lora-finetuned")

Integration with Supervision for Visualization

The Supervision library from Roboflow provides powerful visualization utilities for Florence-2 detection results.

import supervision as sv
import numpy as np

def florencetosupervision(result, taskkey, imagesize):
    """Convert Florence-2 results to Supervision format."""
    detectionsdata = result[taskkey]

    if "bboxes" not in detectionsdata or len(detectionsdata["bboxes"]) == 0:
        return sv.Detections.empty()

    bboxes = np.array(detectionsdata["bboxes"])

    labels = detectionsdata["labels"]

    # Create unique label to classid mapping

    uniquelabels = list(set(labels))
    classids = np.array([uniquelabels.index(l) for l in labels])

    detections = sv.Detections(
        xyxy=bboxes,
        classid=classids,
        data={"classname": np.array(labels)}

    )

    return detections, uniquelabels

Object Detection Visualization

def visualizewithsupervision(image, result, taskkey=""):
    """Visualize detections using Supervision."""
    imagenp = np.array(image)

    detections, labels = florencetosupervision(
        result, taskkey, (image.width, image.height)

    )

    # Bounding box annotator
    boxannotator = sv.BoxAnnotator(
        thickness=2,
        color=sv.ColorPalette.fromhex(

            ["#FF6B6B", "#4ECDC4", "#45B7D1", "#96CEB4", "#FFEAA7"]
        )
    )

    # Label annotator
    labelannotator = sv.LabelAnnotator(
        textscale=0.5,

        textthickness=1,
        textpadding=5,

        color=sv.ColorPalette.fromhex(
            ["#FF6B6B", "#4ECDC4", "#45B7D1", "#96CEB4", "#FFEAA7"]
        )
    )

    # Apply annotations
    annotated = boxannotator.annotate(

        scene=imagenp.copy(),
        detections=detections
    )
    annotated = labelannotator.annotate(

        scene=annotated,
        detections=detections,
        labels=list(detections.data["classname"])
    )

    # Save result
    resultimage = Image.fromarray(annotated)

    resultimage.save("supervisiondetection.png")

    print("Result saved to supervisiondetection.png")

    return annotated

Run detection and visualize
result = runflorence(image, "")

annotated = visualizewithsupervision(image, result)

def visualizeheatmap(image, result, taskkey=""):
    """Create a heatmap from object detections."""
    imagenp = np.array(image)

    detections,  = florencetosupervision(

        result, taskkey, (image.width, image.height)
    )

    heatmapannotator = sv.HeatMapAnnotator(

        radius=40,
        kernelsize=25,
    )

    heatmap = heatmapannotator.annotate(

        scene=imagenp.copy(),
        detections=detections
    )

    resultimage = Image.fromarray(heatmap)

    resultimage.save("detectionheatmap.png")

    return heatmap

result = runflorence(image, "")
heatmap = visualizeheatmap(image, result)

import json
from datetime import datetime
from pathlib import Path

class DocumentAnalysisPipeline:
    """Document analysis pipeline using Florence-2."""

    def init(self, modelid="microsoft/Florence-2-large"):
        self.processor = AutoProcessor.frompretrained(

            modelid, trustremotecode=True
        )
        self.model = AutoModelForCausalLM.frompretrained(

            modelid,
            torchdtype=torch.float16,

            trustremotecode=True

        ).to("cuda")

        self.model.eval()
        print(f"Pipeline loaded with model: {modelid}")

    def runtask(self, image, taskprompt, textinput=None):
        """Run a single Florence-2 task."""
        prompt = taskprompt + textinput if textinput else taskprompt

        inputs = self.processor(
            text=prompt, images=image,
            returntensors="pt"

        ).to("cuda", torch.float16)

        with torch.nograd():
            generatedids = self.model.generate(

                inputids=inputs["inputids"],

                pixelvalues=inputs["pixelvalues"],

                maxnewtokens=1024,

                numbeams=3
            )

        generatedtext = self.processor.batchdecode(
            generatedids, skipspecialtokens=False

        )[0]

        return self.processor.postprocessgeneration(

            generatedtext,
            task=taskprompt,

            imagesize=(image.width, image.height)
        )

    def extracttext(self, image):

        """Extract text from a document."""
        result = self.runtask(image, "WITHREGION>")

        ocrdata = result["WITHREGION>"]

        textblocks = []

        for quad, text in zip(ocrdata["quadboxes"], ocrdata["labels"]):
            textblocks.append({

                "text": text,
                "position": {
                    "x1": quad[0], "y1": quad[1],
                    "x2": quad[2], "y2": quad[3],
                    "x3": quad[4], "y3": quad[5],
                    "x4": quad[6], "y4": quad[7],
                }
            })

        # Sort top-to-bottom, left-to-right
        textblocks.sort(key=lambda b: (b["position"]["y1"], b["position"]["x1"]))

        fulltext = " ".join([b["text"] for b in textblocks])
        return {"fulltext": fulltext, "textblocks": textblocks}

    def generatedescription(self, image):

        """Generate a detailed description of the document."""
        result = self.runtask(image, "DETAILEDCAPTION>")

        return result["DETAILEDCAPTION>"]


    def detectelements(self, image):
        """Detect elements in the document."""
        result = self.runtask(image, "")
        detection = result[""]

        elements = []
        for bbox, label in zip(detection["bboxes"], detection["labels"]):
            elements.append({
                "type": label,
                "bbox": {
                    "x1": bbox[0], "y1": bbox[1],
                    "x2": bbox[2], "y2": bbox[3]
                },
                "area": (bbox[2] - bbox[0])  (bbox[3] - bbox[1])
            })

        return elements

    def detectspecificelements(self, image, elementtypes):

        """Detect specific elements using open-vocabulary detection."""
        textinput = ". ".join(elementtypes) + "."

        result = self.runtask(

            image, "VOCABULARYDETECTION>", textinput=textinput

        )
        return result["VOCABULARYDETECTION>"]


    def analyzedocument(self, imagepath, outputdir="analysisoutput"):

        """Run complete analysis on a single document."""
        os.makedirs(outputdir, existok=True)


        image = loadimage(imagepath)

        print(f"Analyzing: {imagepath}")
        print(f"Image size: {image.width}x{image.height}")

        results = {}

        # Step 1: Extract text
        print("\n[1/4] Extracting text...")
        textresult = self.extracttext(image)
        results["text"] = textresult

        print(f"  Text found: {len(textresult['textblocks'])} blocks")

        print(f"  Preview: {textresult['fulltext'][:200]}...")


        # Step 2: Document description
        print("\n[2/4] Generating description...")
        description = self.generatedescription(image)
        results["description"] = description
        print(f"  Description: {description[:200]}...")

        # Step 3: Element detection
        print("\n[3/4] Detecting elements...")
        elements = self.detectelements(image)

        results["elements"] = elements
        print(f"  Elements detected: {len(elements)}")
        for elem in elements:
            print(f"    - {elem['type']} (area: {elem['area']:.0f}px)")

        # Step 4: Specific document element detection
        print("\n[4/4] Detecting specific document elements...")
        docelements = self.detectspecificelements(
            image,
            ["table", "chart", "logo", "signature", "header", "paragraph"]
        )
        results["documentelements"] = docelements

        # Save results
        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")

        outputfile = os.path.join(outputdir, f"analysis{timestamp}.json")
        with open(outputfile, "w", encoding="utf-8") as f:

            json.dump(results, f, indent=2, ensureascii=False)
        print(f"\nResults saved to: {outputfile}")


        # Create visualization