Complete Guide to PyTorch Lightning: Deep Learning Made Simple
PyTorch Lightning is a high-level framework for PyTorch that simplifies the deep learning model training process. Lightning separates science code from engineering code, making code cleaner, more scalable, and reproducible.
In this tutorial, we'll learn PyTorch Lightning from basics to advanced usage with practical examples.
Why PyTorch Lightning?
PyTorch Lightning Advantages:
Comparison: PyTorch vs PyTorch Lightning
| Aspect | Pure PyTorch | PyTorch Lightning |
|--------|--------------|-------------------|
| Training Loop | Manual | Automatic |
| Multi-GPU | Manual implementation | 1 line change |
| Mixed Precision | Manual setup | Simple flag |
| Checkpointing | Manual | Built-in |
| Logging | Manual | Integrated |
| Code Organization | Free-form | Structured |
Installation
Install PyTorch Lightning
# Install with pip
pip install lightning
Or install with conda
conda install lightning -c conda-forge
Install with extras (for logging, etc)
pip install lightning[extra]
Specific version
pip install lightning==2.1.0
Verify Installation
import lightning as L
import torch
print(f"Lightning version: {L.version}")
print(f"PyTorch version: {torch.version}")
print(f"CUDA available: {torch.cuda.isavailable()}")
Core Concepts
Lightning Structure
PyTorch Lightning
├── LightningModule # Model + Training Logic
├── LightningDataModule # Data Loading
├── Trainer # Training Orchestration
├── Callbacks # Custom Behaviors
└── Loggers # Experiment Tracking
1. LightningModule
LightningModule is the core abstraction that combines model and training logic:
import lightning as L
import torch
import torch.nn as nn
import torch.nn.functional as F
class LitModel(L.LightningModule):
def init(self, inputsize, hiddensize, numclasses, learningrate=1e-3):
super().init()
# Save hyperparameters
self.savehyperparameters()
# Define model architecture
self.layer1 = nn.Linear(inputsize, hiddensize)
self.layer2 = nn.Linear(hiddensize, hiddensize)
self.layer3 = nn.Linear(hiddensize, numclasses)
self.dropout = nn.Dropout(0.2)
def forward(self, x):
"""Forward pass for inference."""
x = F.relu(self.layer1(x))
x = self.dropout(x)
x = F.relu(self.layer2(x))
x = self.dropout(x)
x = self.layer3(x)
return x
def trainingstep(self, batch, batchidx):
"""Single training step."""
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
# Log metrics
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('trainloss', loss, progbar=True)
self.log('trainacc', acc, progbar=True)
return loss
def validationstep(self, batch, batchidx):
"""Single validation step."""
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('valloss', loss, progbar=True)
self.log('valacc', acc, progbar=True)
return loss
def teststep(self, batch, batchidx):
"""Single test step."""
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('testloss', loss)
self.log('testacc', acc)
return loss
def configureoptimizers(self):
"""Configure optimizer and scheduler."""
optimizer = torch.optim.Adam(self.parameters(), lr=self.hparams.learningrate)
scheduler = torch.optim.lrscheduler.StepLR(optimizer, stepsize=10, gamma=0.1)
return {
'optimizer': optimizer,
'lrscheduler': {
'scheduler': scheduler,
'monitor': 'valloss'
}
}
2. LightningDataModule
LightningDataModule organizes all data processing:
from torch.utils.data import DataLoader, randomsplit
from torchvision import transforms
from torchvision.datasets import MNIST
class MNISTDataModule(L.LightningDataModule):
def init(self, datadir='./data', batchsize=32, numworkers=4):
super().init()
self.datadir = datadir
self.batchsize = batchsize
self.numworkers = numworkers
self.transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
def preparedata(self):
"""Download data (called once, single process)."""
MNIST(self.datadir, train=True, download=True)
MNIST(self.datadir, train=False, download=True)
def setup(self, stage=None):
"""Setup datasets (called on every GPU)."""
if stage == 'fit' or stage is None:
mnistfull = MNIST(self.datadir, train=True, transform=self.transform)
self.mnisttrain, self.mnistval = randomsplit(mnistfull, [55000, 5000])
if stage == 'test' or stage is None:
self.mnisttest = MNIST(self.datadir, train=False, transform=self.transform)
def traindataloader(self):
return DataLoader(
self.mnisttrain,
batchsize=self.batchsize,
shuffle=True,
numworkers=self.numworkers,
pinmemory=True
)
def valdataloader(self):
return DataLoader(
self.mnistval,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True
)
def testdataloader(self):
return DataLoader(
self.mnisttest,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True
)
3. Trainer
Trainer handles all training engineering:
# Basic Trainer
trainer = L.Trainer(maxepochs=10)
Trainer with various options
trainer = L.Trainer(
maxepochs=100,
accelerator='gpu', # 'cpu', 'gpu', 'tpu', 'auto'
devices=1, # Number of devices
precision='16-mixed', # Mixed precision training
gradientclipval=1.0, # Gradient clipping
accumulategradbatches=4, # Gradient accumulation
logeverynsteps=10, # Logging frequency
checkvaleverynepoch=1, # Validation frequency
enableprogressbar=True, # Progress bar
enablemodelsummary=True, # Model summary
)
Training
model = LitModel(inputsize=784, hiddensize=256, numclasses=10)
datamodule = MNISTDataModule()
trainer.fit(model, datamodule)
Complete Example: Image Classification
1. CNN Model with Lightning
import lightning as L
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms
from torchvision.datasets import CIFAR10
from torch.utils.data import DataLoader, randomsplit
class LitCNN(L.LightningModule):
def init(self, numclasses=10, learningrate=1e-3):
super().init()
self.savehyperparameters()
# CNN Architecture
self.conv1 = nn.Conv2d(3, 32, 3, padding=1)
self.conv2 = nn.Conv2d(32, 64, 3, padding=1)
self.conv3 = nn.Conv2d(64, 128, 3, padding=1)
self.pool = nn.MaxPool2d(2, 2)
self.fc1 = nn.Linear(128 4 4, 512)
self.fc2 = nn.Linear(512, numclasses)
self.dropout = nn.Dropout(0.5)
# For tracking metrics
self.trainingstepoutputs = []
self.validationstepoutputs = []
def forward(self, x):
x = self.pool(F.relu(self.conv1(x)))
x = self.pool(F.relu(self.conv2(x)))
x = self.pool(F.relu(self.conv3(x)))
x = x.view(-1, 128 4 4)
x = self.dropout(F.relu(self.fc1(x)))
x = self.fc2(x)
return x
def trainingstep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = (preds == y).float().mean()
self.log('trainloss', loss, onstep=True, onepoch=True, progbar=True)
self.log('trainacc', acc, onstep=False, onepoch=True, progbar=True)
return loss
def validationstep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = (preds == y).float().mean()
self.log('valloss', loss, onstep=False, onepoch=True, progbar=True)
self.log('valacc', acc, onstep=False, onepoch=True, progbar=True)
return {'valloss': loss, 'valacc': acc}
def teststep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = (preds == y).float().mean()
self.log('testloss', loss)
self.log('testacc', acc)
return {'testloss': loss, 'testacc': acc}
def predictstep(self, batch, batchidx):
x, = batch
logits = self(x)
preds = torch.argmax(logits, dim=1)
return preds
def configureoptimizers(self):
optimizer = torch.optim.AdamW(
self.parameters(),
lr=self.hparams.learningrate,
weightdecay=1e-4
)
scheduler = torch.optim.lrscheduler.CosineAnnealingLR(
optimizer,
Tmax=100,
etamin=1e-6
)
return {
'optimizer': optimizer,
'lrscheduler': {
'scheduler': scheduler,
'interval': 'epoch',
'frequency': 1
}
}
class CIFAR10DataModule(L.LightningDataModule):
def init(self, datadir='./data', batchsize=64, numworkers=4):
super().init()
self.datadir = datadir
self.batchsize = batchsize
self.numworkers = numworkers
# Augmentation for training
self.traintransform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(brightness=0.2, contrast=0.2),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))
])
# Transform for validation/test
self.testtransform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))
])
def preparedata(self):
CIFAR10(self.datadir, train=True, download=True)
CIFAR10(self.datadir, train=False, download=True)
def setup(self, stage=None):
if stage == 'fit' or stage is None:
cifarfull = CIFAR10(self.datadir, train=True, transform=self.traintransform)
self.cifartrain, self.cifarval = randomsplit(cifarfull, [45000, 5000])
# Apply test transform to validation set
self.cifarval.dataset.transform = self.testtransform
if stage == 'test' or stage is None:
self.cifartest = CIFAR10(self.datadir, train=False, transform=self.testtransform)
def traindataloader(self):
return DataLoader(
self.cifartrain,
batchsize=self.batchsize,
shuffle=True,
numworkers=self.numworkers,
pinmemory=True,
persistentworkers=True
)
def valdataloader(self):
return DataLoader(
self.cifarval,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True,
persistentworkers=True
)
def testdataloader(self):
return DataLoader(
self.cifartest,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True
)
2. Training Script
import lightning as L
from lightning.pytorch.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor
from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger
def main():
# Set seed for reproducibility
L.seedeverything(42)
# Initialize model and data
model = LitCNN(numclasses=10, learningrate=1e-3)
datamodule = CIFAR10DataModule(batchsize=128)
# Callbacks
checkpointcallback = ModelCheckpoint(
dirpath='checkpoints/',
filename='cifar10-{epoch:02d}-{valacc:.4f}',
savetopk=3,
monitor='valacc',
mode='max'
)
earlystopcallback = EarlyStopping(
monitor='valloss',
patience=10,
mode='min'
)
lrmonitor = LearningRateMonitor(logginginterval='step')
# Logger
logger = TensorBoardLogger('logs/', name='cifar10')
# Trainer
trainer = L.Trainer(
maxepochs=100,
accelerator='auto',
devices='auto',
precision='16-mixed',
callbacks=[checkpointcallback, earlystopcallback, lrmonitor],
logger=logger,
gradientclipval=1.0,
accumulategradbatches=1,
logeverynsteps=10,
deterministic=True
)
# Train
trainer.fit(model, datamodule)
# Test
trainer.test(model, datamodule, ckptpath='best')
if name == 'main':
main()
Callbacks
Built-in Callbacks
from lightning.pytorch.callbacks import (
ModelCheckpoint,
EarlyStopping,
LearningRateMonitor,
RichProgressBar,
TQDMProgressBar,
DeviceStatsMonitor,
ModelSummary,
GradientAccumulationScheduler,
StochasticWeightAveraging
)
Model Checkpoint - Save best models
checkpointcallback = ModelCheckpoint(
dirpath='checkpoints/',
filename='{epoch}-{valloss:.2f}',
savetopk=3,
monitor='valloss',
mode='min',
savelast=True,
saveweightsonly=False,
everynepochs=1,
saveontrainepochend=False
)
Early Stopping
earlystop = EarlyStopping(
monitor='valloss',
mindelta=0.001,
patience=10,
verbose=True,
mode='min',
strict=True
)
Learning Rate Monitor
lrmonitor = LearningRateMonitor(logginginterval='step')
Stochastic Weight Averaging
swa = StochasticWeightAveraging(
swalrs=1e-4,
swaepochstart=0.8, # Start SWA at 80% of training
annealingepochs=10
)
Gradient Accumulation Scheduler
gradaccum = GradientAccumulationScheduler(
scheduling={0: 8, 4: 4, 8: 1} # epoch: accumulationfactor
)
Custom Callback
from lightning.pytorch.callbacks import Callback
import torch
class PrintCallback(Callback):
def ontrainstart(self, trainer, plmodule):
print("Training started!")
def ontrainend(self, trainer, plmodule):
print("Training finished!")
def ontrainepochstart(self, trainer, plmodule):
print(f"Epoch {trainer.currentepoch} started")
def ontrainepochend(self, trainer, plmodule):
# Access logged metrics
metrics = trainer.callbackmetrics
print(f"Epoch {trainer.currentepoch} - Loss: {metrics.get('trainloss', 0):.4f}")
class GradientNormCallback(Callback):
"""Log gradient norms for debugging."""
def onbeforeoptimizerstep(self, trainer, plmodule, optimizer):
# Compute gradient norm
totalnorm = 0.0
for p in plmodule.parameters():
if p.grad is not None:
paramnorm = p.grad.data.norm(2)
totalnorm += paramnorm.item() 2
totalnorm = totalnorm 0.5
plmodule.log('gradientnorm', totalnorm)
class ConfusionMatrixCallback(Callback):
"""Log confusion matrix at end of validation."""
def init(self, numclasses):
super().init()
self.numclasses = numclasses
self.preds = []
self.targets = []
def onvalidationbatchend(self, trainer, plmodule, outputs, batch, batchidx):
x, y = batch
logits = plmodule(x)
preds = torch.argmax(logits, dim=1)
self.preds.extend(preds.cpu().numpy())
self.targets.extend(y.cpu().numpy())
def onvalidationepochend(self, trainer, plmodule):
from sklearn.metrics import confusionmatrix
import numpy as np
cm = confusionmatrix(self.targets, self.preds, labels=range(self.numclasses))
print(f"\nConfusion Matrix:\n{cm}")
# Reset
self.preds = []
self.targets = []
Logging
TensorBoard Logger
from lightning.pytorch.loggers import TensorBoardLogger
logger = TensorBoardLogger(
savedir='logs/',
name='myexperiment',
version='v1',
loggraph=True,
defaulthpmetric=False
)
trainer = L.Trainer(logger=logger)
View logs
tensorboard --logdir logs/
Weights & Biases Logger
from lightning.pytorch.loggers import WandbLogger
logger = WandbLogger(
project='my-project',
name='experiment-1',
savedir='logs/',
logmodel='all', # Log model checkpoints
tags=['cnn', 'cifar10']
)
Log custom metrics in LightningModule
class LitModel(L.LightningModule):
def trainingstep(self, batch, batchidx):
# ...
# Log images
if batchidx == 0:
self.logger.experiment.log({
'sampleimages': [wandb.Image(x[0])]
})
CSV Logger
from lightning.pytorch.loggers import CSVLogger
logger = CSVLogger(
savedir='logs/',
name='myexperiment',
version='v1'
)
Multiple Loggers
from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger, CSVLogger
tblogger = TensorBoardLogger('logs/', name='tensorboard')
wandblogger = WandbLogger(project='my-project')
csvlogger = CSVLogger('logs/', name='csv')
trainer = L.Trainer(
logger=[tblogger, wandblogger, csvlogger]
)
Multi-GPU and Distributed Training
Single GPU
trainer = L.Trainer(
accelerator='gpu',
devices=1
)
Multi-GPU (Data Parallel)
# DDP (Distributed Data Parallel) - Recommended
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy='ddp'
)
DDP with find unused parameters
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy='ddpfindunusedparameterstrue'
)
Multi-Node Training
# Multi-node DDP
trainer = L.Trainer(
accelerator='gpu',
devices=8,
numnodes=4,
strategy='ddp'
)
FSDP (Fully Sharded Data Parallel)
from lightning.pytorch.strategies import FSDPStrategy
strategy = FSDPStrategy(
shardingstrategy="FULLSHARD",
cpuoffload=True
)
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy=strategy
)
DeepSpeed Integration
from lightning.pytorch.strategies import DeepSpeedStrategy
strategy = DeepSpeedStrategy(
stage=2,
offloadoptimizer=True,
offloadparameters=True
)
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy=strategy,
precision='16-mixed'
)
Mixed Precision Training
# FP16 Mixed Precision
trainer = L.Trainer(
precision='16-mixed'
)
BF16 Mixed Precision (Ampere GPUs)
trainer = L.Trainer(
precision='bf16-mixed'
)
FP32 (default)
trainer = L.Trainer(
precision='32-true'
)
Checkpointing and Resume Training
Save Checkpoint
from lightning.pytorch.callbacks import ModelCheckpoint
checkpointcallback = ModelCheckpoint(
dirpath='checkpoints/',
filename='model-{epoch:02d}-{valloss:.2f}',
savetopk=3,
monitor='valloss',
mode='min',
savelast=True,
saveweightsonly=False
)
trainer = L.Trainer(callbacks=[checkpointcallback])
Load Checkpoint
# Load for inference
model = LitModel.loadfromcheckpoint('checkpoints/model.ckpt')
model.eval()
Load with different hyperparameters
model = LitModel.loadfromcheckpoint(
'checkpoints/model.ckpt',
learningrate=1e-4 # Override saved hparam
)
Resume training
trainer = L.Trainer()
trainer.fit(model, datamodule, ckptpath='checkpoints/last.ckpt')
Manual Checkpoint
class LitModel(L.LightningModule):
def onsavecheckpoint(self, checkpoint):
# Add custom data to checkpoint
checkpoint['customdata'] = self.customdata
def onloadcheckpoint(self, checkpoint):
# Load custom data from checkpoint
self.customdata = checkpoint['customdata']
Transfer Learning
Fine-tuning Pretrained Model
import lightning as L
import torch
import torch.nn as nn
from torchvision import models
class LitTransferLearning(L.LightningModule):
def init(self, numclasses, learningrate=1e-3, freezebackbone=True):
super().init()
self.savehyperparameters()
# Load pretrained ResNet
self.backbone = models.resnet50(weights=models.ResNet50Weights.IMAGENET1KV2)
# Freeze backbone
if freezebackbone:
for param in self.backbone.parameters():
param.requiresgrad = False
# Replace classifier
numfeatures = self.backbone.fc.infeatures
self.backbone.fc = nn.Sequential(
nn.Linear(numfeatures, 512),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(512, numclasses)
)
def forward(self, x):
return self.backbone(x)
def trainingstep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('trainloss', loss, progbar=True)
self.log('trainacc', acc, progbar=True)
return loss
def configureoptimizers(self):
# Different learning rates for backbone and classifier
backboneparams = [p for n, p in self.backbone.namedparameters()
if 'fc' not in n and p.requiresgrad]
classifierparams = self.backbone.fc.parameters()
optimizer = torch.optim.Adam([
{'params': backboneparams, 'lr': self.hparams.learningrate / 10},
{'params': classifierparams, 'lr': self.hparams.learningrate}
])
scheduler = torch.optim.lrscheduler.CosineAnnealingLR(optimizer, Tmax=50)
return [optimizer], [scheduler]
def ontrainepochstart(self):
# Unfreeze backbone after epoch 5
if self.currentepoch == 5:
for param in self.backbone.parameters():
param.requiresgrad = True
print("Backbone unfrozen!")
Hyperparameter Tuning with Optuna
import optuna
from optuna.integration import PyTorchLightningPruningCallback
def objective(trial):
# Suggest hyperparameters
learningrate = trial.suggestfloat('learningrate', 1e-5, 1e-2, log=True)
batchsize = trial.suggestcategorical('batchsize', [32, 64, 128])
hiddensize = trial.suggestint('hiddensize', 128, 512, step=64)
dropout = trial.suggestfloat('dropout', 0.1, 0.5)
# Create model
model = LitModel(
hiddensize=hiddensize,
learningrate=learningrate,
dropout=dropout
)
# Create data module
datamodule = MyDataModule(batchsize=batchsize)
# Trainer with pruning callback
trainer = L.Trainer(
maxepochs=20,
accelerator='gpu',
devices=1,
callbacks=[PyTorchLightningPruningCallback(trial, monitor='valloss')],
enableprogressbar=False,
enablemodelsummary=False
)
trainer.fit(model, datamodule)
return trainer.callbackmetrics['valloss'].item()
Run optimization
study = optuna.createstudy(direction='minimize')
study.optimize(objective, ntrials=100, timeout=3600)
print(f"Best trial: {study.besttrial.params}")
Inference and Deployment
Basic Inference
# Load model
model = LitModel.loadfromcheckpoint('checkpoints/best.ckpt')
model.eval()
model.freeze()
Single prediction
with torch.nograd():
x = torch.randn(1, 3, 32, 32)
output = model(x)
pred = output.argmax(dim=1)
Batch Prediction
# Using Trainer for batch prediction
trainer = L.Trainer(accelerator='gpu', devices=1)
predictions = trainer.predict(model, datamodule)
predictions is a list of batch predictions
allpreds = torch.cat(predictions, dim=0)
Export to ONNX
import torch.onnx
model = LitModel.loadfromcheckpoint('checkpoints/best.ckpt')
model.eval()
Dummy input
dummyinput = torch.randn(1, 3, 32, 32)
Export
torch.onnx.export(
model,
dummyinput,
'model.onnx',
inputnames=['input'],
outputnames=['output'],
dynamicaxes={
'input': {0: 'batchsize'},
'output': {0: 'batchsize'}
},
opsetversion=11
)
Export to TorchScript
model = LitModel.loadfromcheckpoint('checkpoints/best.ckpt')
model.eval()
Script model
script
model = model.totorchscript()
script
model.save('modelscripted.pt')
Load and use
loaded
model = torch.jit.load('modelscripted.pt')
output = loaded
model(torch.randn(1, 3, 32, 32))
Best Practices
1. Code Organization
# Good project structure
project/
├── configs/
│ └── config.yaml
├── data/
│ └── datamodule.py
├── models/
│ └── model.py
├── callbacks/
│ └── customcallbacks.py
├── utils/
│ └── helpers.py
├── train.py
├── test.py
└── requirements.txt
2. Configuration with Hydra
import hydra
from omegaconf import DictConfig
@hydra.main(configpath='configs', configname='config')
def main(cfg: DictConfig):
model = LitModel(
hiddensize=cfg.model.hiddensize,
learningrate=cfg.training.learningrate
)
trainer = L.Trainer(
maxepochs=cfg.training.maxepochs,
accelerator=cfg.training.accelerator
)
trainer.fit(model, datamodule)
if name == 'main':
main()
3. Reproducibility
import lightning as L
Set seed
L.seedeverything(42, workers=True)
Deterministic training
trainer = L.Trainer(
deterministic=True,
benchmark=False
)
4. Memory Optimization
# Gradient checkpointing
class LitModel(L.LightningModule):
def init(self):
super().init()
# Enable gradient checkpointing for backbone
self.backbone = models.resnet50()
self.backbone.layer3.apply(self.enablegradientcheckpointing)
def enablegradientcheckpointing(self, module):
if hasattr(module, 'gradientcheckpointing'):
module.gradientcheckpointing = True
Clear cache periodically
trainer = L.Trainer(
callbacks=[
L.pytorch.callbacks.DeviceStatsMonitor()
]
)
5. Profiling
from lightning.pytorch.profilers import AdvancedProfiler, PyTorchProfiler
Simple profiler
trainer = L.Trainer(profiler='simple')
Advanced profiler
profiler = AdvancedProfiler(dirpath='profiler/', filename='perflogs')
trainer = L.Trainer(profiler=profiler)
PyTorch profiler
profiler = PyTorchProfiler(
dirpath='profiler/',
filename='pytorchprofiler',
schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2),
ontraceready=torch.profiler.tensorboardtracehandler('logs/profiler')
)
trainer = L.Trainer(profiler=profiler)
Troubleshooting
1. Out of Memory
# Reduce batch size
datamodule = MyDataModule(batchsize=16)
Gradient accumulation
trainer = L.Trainer(accumulategradbatches=4)
Mixed precision
trainer = L.Trainer(precision='16-mixed')
Gradient checkpointing in model
2. Slow Training
# Increase numworkers
datamodule = MyDataModule(numworkers=8)
Pin memory
DataLoader(..., pinmemory=True)
Use persistent workers
DataLoader(..., persistentworkers=True)
Profile to identify bottleneck
trainer = L.Trainer(profiler='simple')
3. NaN Loss
# Check for NaN
class LitModel(L.LightningModule):
def trainingstep(self, batch, batchidx):
loss = ...
if torch.isnan(loss):
print(f"NaN loss at batch {batchidx}")
return None
return loss
Gradient clipping
trainer = L.Trainer(gradientclip_val=1.0)
Lower learning rate
Use gradient scaling with mixed precision
Conclusion
PyTorch Lightning is a powerful framework for deep learning:
Key Takeaways:
When to Use PyTorch Lightning:
Use Lightning when:- Serious deep learning projects
- Need scalability (multi-GPU, distributed)
- Teams with multiple researchers
- Production deployment
- Experiment tracking is important
- Simple prototyping
- Educational purposes
- Very custom training loops
- Minimal dependencies needed
Resources:
- Docs: https://lightning.ai/docs/pytorch/stable/
- GitHub: https://github.com/Lightning-AI/lightning
- Examples: https://github.com/Lightning-AI/lightning/tree/master/examples