Panduan Lengkap PyTorch Lightning: Deep Learning Made Simple
PyTorch Lightning adalah framework high-level untuk PyTorch yang menyederhanakan proses training deep learning model. Lightning memisahkan science code dari engineering code, membuat kode lebih clean, scalable, dan reproducible.
Dalam tutorial ini, kita akan mempelajari PyTorch Lightning dari dasar hingga advanced usage dengan contoh praktis.
Mengapa PyTorch Lightning?
Keunggulan PyTorch Lightning:
Perbandingan: PyTorch vs PyTorch Lightning
| Aspek | PyTorch Murni | PyTorch Lightning |
|-------|---------------|-------------------|
| Training Loop | Manual | Otomatis |
| Multi-GPU | Implementasi manual | 1 line change |
| Mixed Precision | Setup manual | Flag sederhana |
| Checkpointing | Manual | Built-in |
| Logging | Manual | Terintegrasi |
| Code Organization | Bebas | Terstruktur |
Instalasi
Install PyTorch Lightning
# Install dengan pip
pip install lightning
Atau install dengan conda
conda install lightning -c conda-forge
Install dengan extras (untuk logging, etc)
pip install lightning[extra]
Versi spesifik
pip install lightning==2.1.0
Verifikasi Instalasi
import lightning as L
import torch
print(f"Lightning version: {L.version}")
print(f"PyTorch version: {torch.version}")
print(f"CUDA available: {torch.cuda.isavailable()}")
Konsep Dasar
Struktur Lightning
PyTorch Lightning
├── LightningModule # Model + Training Logic
├── LightningDataModule # Data Loading
├── Trainer # Training Orchestration
├── Callbacks # Custom Behaviors
└── Loggers # Experiment Tracking
1. LightningModule
LightningModule adalah core abstraction yang menggabungkan model dan training logic:
import lightning as L
import torch
import torch.nn as nn
import torch.nn.functional as F
class LitModel(L.LightningModule):
def init(self, inputsize, hiddensize, numclasses, learningrate=1e-3):
super().init()
# Save hyperparameters
self.savehyperparameters()
# Define model architecture
self.layer1 = nn.Linear(inputsize, hiddensize)
self.layer2 = nn.Linear(hiddensize, hiddensize)
self.layer3 = nn.Linear(hiddensize, numclasses)
self.dropout = nn.Dropout(0.2)
def forward(self, x):
"""Forward pass untuk inference."""
x = F.relu(self.layer1(x))
x = self.dropout(x)
x = F.relu(self.layer2(x))
x = self.dropout(x)
x = self.layer3(x)
return x
def trainingstep(self, batch, batchidx):
"""Single training step."""
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
# Log metrics
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('trainloss', loss, progbar=True)
self.log('trainacc', acc, progbar=True)
return loss
def validationstep(self, batch, batchidx):
"""Single validation step."""
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('valloss', loss, progbar=True)
self.log('valacc', acc, progbar=True)
return loss
def teststep(self, batch, batchidx):
"""Single test step."""
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('testloss', loss)
self.log('testacc', acc)
return loss
def configureoptimizers(self):
"""Configure optimizer and scheduler."""
optimizer = torch.optim.Adam(self.parameters(), lr=self.hparams.learningrate)
scheduler = torch.optim.lrscheduler.StepLR(optimizer, stepsize=10, gamma=0.1)
return {
'optimizer': optimizer,
'lrscheduler': {
'scheduler': scheduler,
'monitor': 'valloss'
}
}
2. LightningDataModule
LightningDataModule mengorganisasi semua data processing:
from torch.utils.data import DataLoader, randomsplit
from torchvision import transforms
from torchvision.datasets import MNIST
class MNISTDataModule(L.LightningDataModule):
def init(self, datadir='./data', batchsize=32, numworkers=4):
super().init()
self.datadir = datadir
self.batchsize = batchsize
self.numworkers = numworkers
self.transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
def preparedata(self):
"""Download data (called once, single process)."""
MNIST(self.datadir, train=True, download=True)
MNIST(self.datadir, train=False, download=True)
def setup(self, stage=None):
"""Setup datasets (called on every GPU)."""
if stage == 'fit' or stage is None:
mnistfull = MNIST(self.datadir, train=True, transform=self.transform)
self.mnisttrain, self.mnistval = randomsplit(mnistfull, [55000, 5000])
if stage == 'test' or stage is None:
self.mnisttest = MNIST(self.datadir, train=False, transform=self.transform)
def traindataloader(self):
return DataLoader(
self.mnisttrain,
batchsize=self.batchsize,
shuffle=True,
numworkers=self.numworkers,
pinmemory=True
)
def valdataloader(self):
return DataLoader(
self.mnistval,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True
)
def testdataloader(self):
return DataLoader(
self.mnisttest,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True
)
3. Trainer
Trainer menghandle semua training engineering:
# Basic Trainer
trainer = L.Trainer(maxepochs=10)
Trainer dengan berbagai options
trainer = L.Trainer(
maxepochs=100,
accelerator='gpu', # 'cpu', 'gpu', 'tpu', 'auto'
devices=1, # Jumlah devices
precision='16-mixed', # Mixed precision training
gradientclipval=1.0, # Gradient clipping
accumulategradbatches=4, # Gradient accumulation
logeverynsteps=10, # Logging frequency
checkvaleverynepoch=1, # Validation frequency
enableprogressbar=True, # Progress bar
enablemodelsummary=True, # Model summary
)
Training
model = LitModel(inputsize=784, hiddensize=256, numclasses=10)
datamodule = MNISTDataModule()
trainer.fit(model, datamodule)
Complete Example: Image Classification
1. CNN Model dengan Lightning
import lightning as L
import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import transforms
from torchvision.datasets import CIFAR10
from torch.utils.data import DataLoader, randomsplit
class LitCNN(L.LightningModule):
def init(self, numclasses=10, learningrate=1e-3):
super().init()
self.savehyperparameters()
# CNN Architecture
self.conv1 = nn.Conv2d(3, 32, 3, padding=1)
self.conv2 = nn.Conv2d(32, 64, 3, padding=1)
self.conv3 = nn.Conv2d(64, 128, 3, padding=1)
self.pool = nn.MaxPool2d(2, 2)
self.fc1 = nn.Linear(128 4 4, 512)
self.fc2 = nn.Linear(512, numclasses)
self.dropout = nn.Dropout(0.5)
# For tracking metrics
self.trainingstepoutputs = []
self.validationstepoutputs = []
def forward(self, x):
x = self.pool(F.relu(self.conv1(x)))
x = self.pool(F.relu(self.conv2(x)))
x = self.pool(F.relu(self.conv3(x)))
x = x.view(-1, 128 4 4)
x = self.dropout(F.relu(self.fc1(x)))
x = self.fc2(x)
return x
def trainingstep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = (preds == y).float().mean()
self.log('trainloss', loss, onstep=True, onepoch=True, progbar=True)
self.log('trainacc', acc, onstep=False, onepoch=True, progbar=True)
return loss
def validationstep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = (preds == y).float().mean()
self.log('valloss', loss, onstep=False, onepoch=True, progbar=True)
self.log('valacc', acc, onstep=False, onepoch=True, progbar=True)
return {'valloss': loss, 'valacc': acc}
def teststep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
preds = torch.argmax(logits, dim=1)
acc = (preds == y).float().mean()
self.log('testloss', loss)
self.log('testacc', acc)
return {'testloss': loss, 'testacc': acc}
def predictstep(self, batch, batchidx):
x, = batch
logits = self(x)
preds = torch.argmax(logits, dim=1)
return preds
def configureoptimizers(self):
optimizer = torch.optim.AdamW(
self.parameters(),
lr=self.hparams.learningrate,
weightdecay=1e-4
)
scheduler = torch.optim.lrscheduler.CosineAnnealingLR(
optimizer,
Tmax=100,
etamin=1e-6
)
return {
'optimizer': optimizer,
'lrscheduler': {
'scheduler': scheduler,
'interval': 'epoch',
'frequency': 1
}
}
class CIFAR10DataModule(L.LightningDataModule):
def init(self, datadir='./data', batchsize=64, numworkers=4):
super().init()
self.datadir = datadir
self.batchsize = batchsize
self.numworkers = numworkers
# Augmentation untuk training
self.traintransform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(brightness=0.2, contrast=0.2),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))
])
# Transform untuk validation/test
self.testtransform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))
])
def preparedata(self):
CIFAR10(self.datadir, train=True, download=True)
CIFAR10(self.datadir, train=False, download=True)
def setup(self, stage=None):
if stage == 'fit' or stage is None:
cifarfull = CIFAR10(self.datadir, train=True, transform=self.traintransform)
self.cifartrain, self.cifarval = randomsplit(cifarfull, [45000, 5000])
# Apply test transform to validation set
self.cifarval.dataset.transform = self.testtransform
if stage == 'test' or stage is None:
self.cifartest = CIFAR10(self.datadir, train=False, transform=self.testtransform)
def traindataloader(self):
return DataLoader(
self.cifartrain,
batchsize=self.batchsize,
shuffle=True,
numworkers=self.numworkers,
pinmemory=True,
persistentworkers=True
)
def valdataloader(self):
return DataLoader(
self.cifarval,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True,
persistentworkers=True
)
def testdataloader(self):
return DataLoader(
self.cifartest,
batchsize=self.batchsize,
shuffle=False,
numworkers=self.numworkers,
pinmemory=True
)
2. Training Script
import lightning as L
from lightning.pytorch.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor
from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger
def main():
# Set seed for reproducibility
L.seedeverything(42)
# Initialize model and data
model = LitCNN(numclasses=10, learningrate=1e-3)
datamodule = CIFAR10DataModule(batchsize=128)
# Callbacks
checkpointcallback = ModelCheckpoint(
dirpath='checkpoints/',
filename='cifar10-{epoch:02d}-{valacc:.4f}',
savetopk=3,
monitor='valacc',
mode='max'
)
earlystopcallback = EarlyStopping(
monitor='valloss',
patience=10,
mode='min'
)
lrmonitor = LearningRateMonitor(logginginterval='step')
# Logger
logger = TensorBoardLogger('logs/', name='cifar10')
# Trainer
trainer = L.Trainer(
maxepochs=100,
accelerator='auto',
devices='auto',
precision='16-mixed',
callbacks=[checkpointcallback, earlystopcallback, lrmonitor],
logger=logger,
gradientclipval=1.0,
accumulategradbatches=1,
logeverynsteps=10,
deterministic=True
)
# Train
trainer.fit(model, datamodule)
# Test
trainer.test(model, datamodule, ckptpath='best')
if name == 'main':
main()
Callbacks
Built-in Callbacks
from lightning.pytorch.callbacks import (
ModelCheckpoint,
EarlyStopping,
LearningRateMonitor,
RichProgressBar,
TQDMProgressBar,
DeviceStatsMonitor,
ModelSummary,
GradientAccumulationScheduler,
StochasticWeightAveraging
)
Model Checkpoint - Save best models
checkpointcallback = ModelCheckpoint(
dirpath='checkpoints/',
filename='{epoch}-{valloss:.2f}',
savetopk=3,
monitor='valloss',
mode='min',
savelast=True,
saveweightsonly=False,
everynepochs=1,
saveontrainepochend=False
)
Early Stopping
earlystop = EarlyStopping(
monitor='valloss',
mindelta=0.001,
patience=10,
verbose=True,
mode='min',
strict=True
)
Learning Rate Monitor
lrmonitor = LearningRateMonitor(logginginterval='step')
Stochastic Weight Averaging
swa = StochasticWeightAveraging(
swalrs=1e-4,
swaepochstart=0.8, # Start SWA at 80% of training
annealingepochs=10
)
Gradient Accumulation Scheduler
gradaccum = GradientAccumulationScheduler(
scheduling={0: 8, 4: 4, 8: 1} # epoch: accumulationfactor
)
Custom Callback
from lightning.pytorch.callbacks import Callback
import torch
class PrintCallback(Callback):
def ontrainstart(self, trainer, plmodule):
print("Training started!")
def ontrainend(self, trainer, plmodule):
print("Training finished!")
def ontrainepochstart(self, trainer, plmodule):
print(f"Epoch {trainer.currentepoch} started")
def ontrainepochend(self, trainer, plmodule):
# Access logged metrics
metrics = trainer.callbackmetrics
print(f"Epoch {trainer.currentepoch} - Loss: {metrics.get('trainloss', 0):.4f}")
class GradientNormCallback(Callback):
"""Log gradient norms for debugging."""
def onbeforeoptimizerstep(self, trainer, plmodule, optimizer):
# Compute gradient norm
totalnorm = 0.0
for p in plmodule.parameters():
if p.grad is not None:
paramnorm = p.grad.data.norm(2)
totalnorm += paramnorm.item() 2
totalnorm = totalnorm 0.5
plmodule.log('gradientnorm', totalnorm)
class ConfusionMatrixCallback(Callback):
"""Log confusion matrix at end of validation."""
def init(self, numclasses):
super().init()
self.numclasses = numclasses
self.preds = []
self.targets = []
def onvalidationbatchend(self, trainer, plmodule, outputs, batch, batchidx):
x, y = batch
logits = plmodule(x)
preds = torch.argmax(logits, dim=1)
self.preds.extend(preds.cpu().numpy())
self.targets.extend(y.cpu().numpy())
def onvalidationepochend(self, trainer, plmodule):
from sklearn.metrics import confusionmatrix
import numpy as np
cm = confusionmatrix(self.targets, self.preds, labels=range(self.numclasses))
print(f"\nConfusion Matrix:\n{cm}")
# Reset
self.preds = []
self.targets = []
Logging
TensorBoard Logger
from lightning.pytorch.loggers import TensorBoardLogger
logger = TensorBoardLogger(
savedir='logs/',
name='myexperiment',
version='v1',
loggraph=True,
defaulthpmetric=False
)
trainer = L.Trainer(logger=logger)
View logs
tensorboard --logdir logs/
Weights & Biases Logger
from lightning.pytorch.loggers import WandbLogger
logger = WandbLogger(
project='my-project',
name='experiment-1',
savedir='logs/',
logmodel='all', # Log model checkpoints
tags=['cnn', 'cifar10']
)
Log custom metrics dalam LightningModule
class LitModel(L.LightningModule):
def trainingstep(self, batch, batchidx):
# ...
# Log images
if batchidx == 0:
self.logger.experiment.log({
'sampleimages': [wandb.Image(x[0])]
})
CSV Logger
from lightning.pytorch.loggers import CSVLogger
logger = CSVLogger(
savedir='logs/',
name='myexperiment',
version='v1'
)
Multiple Loggers
from lightning.pytorch.loggers import TensorBoardLogger, WandbLogger, CSVLogger
tblogger = TensorBoardLogger('logs/', name='tensorboard')
wandblogger = WandbLogger(project='my-project')
csvlogger = CSVLogger('logs/', name='csv')
trainer = L.Trainer(
logger=[tblogger, wandblogger, csvlogger]
)
Multi-GPU dan Distributed Training
Single GPU
trainer = L.Trainer(
accelerator='gpu',
devices=1
)
Multi-GPU (Data Parallel)
# DDP (Distributed Data Parallel) - Recommended
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy='ddp'
)
DDP with find unused parameters
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy='ddpfindunusedparameterstrue'
)
Multi-Node Training
# Multi-node DDP
trainer = L.Trainer(
accelerator='gpu',
devices=8,
numnodes=4,
strategy='ddp'
)
FSDP (Fully Sharded Data Parallel)
from lightning.pytorch.strategies import FSDPStrategy
strategy = FSDPStrategy(
shardingstrategy="FULLSHARD",
cpuoffload=True
)
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy=strategy
)
DeepSpeed Integration
from lightning.pytorch.strategies import DeepSpeedStrategy
strategy = DeepSpeedStrategy(
stage=2,
offloadoptimizer=True,
offloadparameters=True
)
trainer = L.Trainer(
accelerator='gpu',
devices=4,
strategy=strategy,
precision='16-mixed'
)
Mixed Precision Training
# FP16 Mixed Precision
trainer = L.Trainer(
precision='16-mixed'
)
BF16 Mixed Precision (Ampere GPUs)
trainer = L.Trainer(
precision='bf16-mixed'
)
FP32 (default)
trainer = L.Trainer(
precision='32-true'
)
Checkpointing dan Resume Training
Save Checkpoint
from lightning.pytorch.callbacks import ModelCheckpoint
checkpointcallback = ModelCheckpoint(
dirpath='checkpoints/',
filename='model-{epoch:02d}-{valloss:.2f}',
savetopk=3,
monitor='valloss',
mode='min',
savelast=True,
saveweightsonly=False
)
trainer = L.Trainer(callbacks=[checkpointcallback])
Load Checkpoint
# Load untuk inference
model = LitModel.loadfromcheckpoint('checkpoints/model.ckpt')
model.eval()
Load dengan hyperparameters berbeda
model = LitModel.loadfromcheckpoint(
'checkpoints/model.ckpt',
learningrate=1e-4 # Override saved hparam
)
Resume training
trainer = L.Trainer()
trainer.fit(model, datamodule, ckptpath='checkpoints/last.ckpt')
Manual Checkpoint
class LitModel(L.LightningModule):
def onsavecheckpoint(self, checkpoint):
# Add custom data to checkpoint
checkpoint['customdata'] = self.customdata
def onloadcheckpoint(self, checkpoint):
# Load custom data from checkpoint
self.customdata = checkpoint['customdata']
Transfer Learning
Fine-tuning Pretrained Model
import lightning as L
import torch
import torch.nn as nn
from torchvision import models
class LitTransferLearning(L.LightningModule):
def init(self, numclasses, learningrate=1e-3, freezebackbone=True):
super().init()
self.savehyperparameters()
# Load pretrained ResNet
self.backbone = models.resnet50(weights=models.ResNet50Weights.IMAGENET1KV2)
# Freeze backbone
if freezebackbone:
for param in self.backbone.parameters():
param.requiresgrad = False
# Replace classifier
numfeatures = self.backbone.fc.infeatures
self.backbone.fc = nn.Sequential(
nn.Linear(numfeatures, 512),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(512, numclasses)
)
def forward(self, x):
return self.backbone(x)
def trainingstep(self, batch, batchidx):
x, y = batch
logits = self(x)
loss = F.crossentropy(logits, y)
acc = (logits.argmax(dim=1) == y).float().mean()
self.log('trainloss', loss, progbar=True)
self.log('trainacc', acc, progbar=True)
return loss
def configureoptimizers(self):
# Different learning rates untuk backbone dan classifier
backboneparams = [p for n, p in self.backbone.namedparameters()
if 'fc' not in n and p.requiresgrad]
classifierparams = self.backbone.fc.parameters()
optimizer = torch.optim.Adam([
{'params': backboneparams, 'lr': self.hparams.learningrate / 10},
{'params': classifierparams, 'lr': self.hparams.learningrate}
])
scheduler = torch.optim.lrscheduler.CosineAnnealingLR(optimizer, Tmax=50)
return [optimizer], [scheduler]
def ontrainepochstart(self):
# Unfreeze backbone after epoch 5
if self.currentepoch == 5:
for param in self.backbone.parameters():
param.requiresgrad = True
print("Backbone unfrozen!")
Hyperparameter Tuning dengan Optuna
import optuna
from optuna.integration import PyTorchLightningPruningCallback
def objective(trial):
# Suggest hyperparameters
learningrate = trial.suggestfloat('learningrate', 1e-5, 1e-2, log=True)
batchsize = trial.suggestcategorical('batchsize', [32, 64, 128])
hiddensize = trial.suggestint('hiddensize', 128, 512, step=64)
dropout = trial.suggestfloat('dropout', 0.1, 0.5)
# Create model
model = LitModel(
hiddensize=hiddensize,
learningrate=learningrate,
dropout=dropout
)
# Create data module
datamodule = MyDataModule(batchsize=batchsize)
# Trainer with pruning callback
trainer = L.Trainer(
maxepochs=20,
accelerator='gpu',
devices=1,
callbacks=[PyTorchLightningPruningCallback(trial, monitor='valloss')],
enableprogressbar=False,
enablemodelsummary=False
)
trainer.fit(model, datamodule)
return trainer.callbackmetrics['valloss'].item()
Run optimization
study = optuna.createstudy(direction='minimize')
study.optimize(objective, ntrials=100, timeout=3600)
print(f"Best trial: {study.besttrial.params}")
Inference dan Deployment
Basic Inference
# Load model
model = LitModel.loadfromcheckpoint('checkpoints/best.ckpt')
model.eval()
model.freeze()
Single prediction
with torch.nograd():
x = torch.randn(1, 3, 32, 32)
output = model(x)
pred = output.argmax(dim=1)
Batch Prediction
# Using Trainer for batch prediction
trainer = L.Trainer(accelerator='gpu', devices=1)
predictions = trainer.predict(model, datamodule)
predictions adalah list of batch predictions
allpreds = torch.cat(predictions, dim=0)
Export ke ONNX
import torch.onnx
model = LitModel.loadfromcheckpoint('checkpoints/best.ckpt')
model.eval()
Dummy input
dummyinput = torch.randn(1, 3, 32, 32)
Export
torch.onnx.export(
model,
dummyinput,
'model.onnx',
inputnames=['input'],
outputnames=['output'],
dynamicaxes={
'input': {0: 'batchsize'},
'output': {0: 'batchsize'}
},
opsetversion=11
)
Export ke TorchScript
model = LitModel.loadfromcheckpoint('checkpoints/best.ckpt')
model.eval()
Script model
script
model = model.totorchscript()
script
model.save('modelscripted.pt')
Load dan use
loaded
model = torch.jit.load('modelscripted.pt')
output = loaded
model(torch.randn(1, 3, 32, 32))
Best Practices
1. Code Organization
# Struktur project yang baik
project/
├── configs/
│ └── config.yaml
├── data/
│ └── datamodule.py
├── models/
│ └── model.py
├── callbacks/
│ └── customcallbacks.py
├── utils/
│ └── helpers.py
├── train.py
├── test.py
└── requirements.txt
2. Configuration dengan Hydra
import hydra
from omegaconf import DictConfig
@hydra.main(configpath='configs', configname='config')
def main(cfg: DictConfig):
model = LitModel(
hiddensize=cfg.model.hiddensize,
learningrate=cfg.training.learningrate
)
trainer = L.Trainer(
maxepochs=cfg.training.maxepochs,
accelerator=cfg.training.accelerator
)
trainer.fit(model, datamodule)
if name == 'main':
main()
3. Reproducibility
import lightning as L
Set seed
L.seedeverything(42, workers=True)
Deterministic training
trainer = L.Trainer(
deterministic=True,
benchmark=False
)
4. Memory Optimization
# Gradient checkpointing
class LitModel(L.LightningModule):
def init(self):
super().init()
# Enable gradient checkpointing untuk backbone
self.backbone = models.resnet50()
self.backbone.layer3.apply(self.enablegradientcheckpointing)
def enablegradientcheckpointing(self, module):
if hasattr(module, 'gradientcheckpointing'):
module.gradientcheckpointing = True
Clear cache periodically
trainer = L.Trainer(
callbacks=[
L.pytorch.callbacks.DeviceStatsMonitor()
]
)
5. Profiling
from lightning.pytorch.profilers import AdvancedProfiler, PyTorchProfiler
Simple profiler
trainer = L.Trainer(profiler='simple')
Advanced profiler
profiler = AdvancedProfiler(dirpath='profiler/', filename='perflogs')
trainer = L.Trainer(profiler=profiler)
PyTorch profiler
profiler = PyTorchProfiler(
dirpath='profiler/',
filename='pytorchprofiler',
schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2),
ontraceready=torch.profiler.tensorboardtracehandler('logs/profiler')
)
trainer = L.Trainer(profiler=profiler)
Troubleshooting
1. Out of Memory
# Reduce batch size
datamodule = MyDataModule(batchsize=16)
Gradient accumulation
trainer = L.Trainer(accumulategradbatches=4)
Mixed precision
trainer = L.Trainer(precision='16-mixed')
Gradient checkpointing di model
2. Slow Training
# Increase numworkers
datamodule = MyDataModule(numworkers=8)
Pin memory
DataLoader(..., pinmemory=True)
Use persistent workers
DataLoader(..., persistentworkers=True)
Profile untuk identify bottleneck
trainer = L.Trainer(profiler='simple')
3. NaN Loss
# Check for NaN
class LitModel(L.LightningModule):
def trainingstep(self, batch, batchidx):
loss = ...
if torch.isnan(loss):
print(f"NaN loss at batch {batchidx}")
return None
return loss
Gradient clipping
trainer = L.Trainer(gradientclip_val=1.0)
Lower learning rate
Use gradient scaling dengan mixed precision
Kesimpulan
PyTorch Lightning adalah framework yang sangat powerful untuk deep learning:
Key Takeaways:
Kapan Menggunakan PyTorch Lightning:
Use Lightning when:- Project deep learning yang serius
- Butuh scalability (multi-GPU, distributed)
- Tim dengan multiple researchers
- Production deployment
- Experiment tracking penting
- Simple prototyping
- Educational purposes
- Sangat custom training loop
- Minimal dependencies needed