Tutorial Lengkap Optuna: Optimasi Hyperparameter Otomatis

Daftar Isi

Pendahuluan

Tuning hyperparameter adalah salah satu tugas yang paling memakan waktu dalam machine learning. Optuna adalah framework optimasi hyperparameter otomatis yang menggunakan algoritma pencarian efisien untuk menemukan konfigurasi optimal secara signifikan lebih cepat dibanding grid search atau random search.

Dalam tutorial ini, Anda akan mempelajari cara menggunakan Optuna mulai dari optimasi dasar hingga tuning multi-objektif terdistribusi di berbagai framework ML.

Prasyarat

• Python 3.8+
• Pemahaman dasar machine learning
• Familiar dengan scikit-learn atau PyTorch

Instalasi dan Setup

pip install optuna optuna-dashboard plotly scikit-learn xgboost lightgbm torch

Opsional: untuk optimasi terdistribusi
pip install optuna[rdb] psycopg2-binary

Konsep Dasar: Study dan Trial

Optuna berpusat pada dua konsep utama:

• Study: Sesi optimasi yang terdiri dari banyak trial
• Trial: Satu evaluasi fungsi objektif dengan hyperparameter tertentu

import optuna

Definisikan fungsi objektif
def fungsiobjektif(trial):

    # Sarankan hyperparameter
    x = trial.suggestfloat("x", -10, 10)
    y = trial.suggestfloat("y", -10, 10)


    # Kembalikan nilai untuk diminimalkan
    return (x - 2)  2 + (y + 3)  2

Buat study dan optimasi
study = optuna.createstudy(direction="minimize")
study.optimize(fungsiobjektif, ntrials=100)

print(f"Nilai terbaik: {study.bestvalue:.4f}")

print(f"Parameter terbaik: {study.bestparams}")

Metode Saran Parameter

def fungsiobjektif(trial):
    # Parameter integer
    jumlahlayer = trial.suggestint("jumlahlayer", 1, 5)

    # Parameter float
    learningrate = trial.suggestfloat("lr", 1e-5, 1e-1, log=True)

    # Parameter kategorikal
    optimizer = trial.suggestcategorical("optimizer", ["adam", "sgd", "rmsprop"])


    # Float diskret
    dropout = trial.suggestfloat("dropout", 0.1, 0.5, step=0.1)

    # Parameter kondisional
    if optimizer == "sgd":
        momentum = trial.suggestfloat("momentum", 0.0, 0.99)


    return latihmodel(jumlahlayer, learningrate, optimizer, dropout)

Optimasi Dasar dengan Scikit-Learn

import optuna
from sklearn.ensemble import RandomForestClassifier
from sklearn.modelselection import crossvalscore

from sklearn.datasets import loadbreastcancer


X, y = loadbreastcancer(returnXy=True)


def fungsiobjektif(trial):
    params = {
        "nestimators": trial.suggestint("nestimators", 50, 500),

        "maxdepth": trial.suggestint("maxdepth", 3, 20),
        "minsamplessplit": trial.suggestint("minsamplessplit", 2, 20),

        "minsamplesleaf": trial.suggestint("minsamplesleaf", 1, 10),
        "maxfeatures": trial.suggestcategorical("maxfeatures", ["sqrt", "log2", None]),

        "criterion": trial.suggestcategorical("criterion", ["gini", "entropy"]),
    }

    clf = RandomForestClassifier(params, randomstate=42, njobs=-1)

    skor = crossvalscore(clf, X, y, cv=5, scoring="accuracy")

    return skor.mean()

study = optuna.createstudy(direction="maximize")

study.optimize(fungsiobjektif, ntrials=200, showprogressbar=True)


print(f"Akurasi terbaik: {study.bestvalue:.4f}")
print(f"Parameter terbaik: {study.bestparams}")

# TPE Sampler (default) - Tree-structured Parzen Estimator
from optuna.samplers import TPESampler
study = optuna.createstudy(sampler=TPESampler(seed=42))

CMA-ES Sampler - untuk parameter kontinu
from optuna.samplers import CmaEsSampler
study = optuna.createstudy(sampler=CmaEsSampler(seed=42))


Grid Sampler - pencarian menyeluruh
from optuna.samplers import GridSampler
ruangpencarian = {
    "lr": [1e-4, 1e-3, 1e-2],
    "jumlahlayer": [1, 2, 3, 4],

}
study = optuna.createstudy(sampler=GridSampler(ruangpencarian))


Random Sampler - sebagai baseline perbandingan
from optuna.samplers import RandomSampler
study = optuna.createstudy(sampler=RandomSampler(seed=42))

QMC Sampler - Quasi-Monte Carlo untuk cakupan lebih baik
from optuna.samplers import QMCSampler
study = optuna.createstudy(sampler=QMCSampler(seed=42))

import time

daftarsampler = {
    "TPE": TPESampler(seed=42),
    "CMA-ES": CmaEsSampler(seed=42),
    "Random": RandomSampler(seed=42),
}

hasil = {}
for nama, sampler in daftarsampler.items():

    study = optuna.createstudy(direction="minimize", sampler=sampler)
    mulai = time.time()
    study.optimize(fungsiobjektif, ntrials=100, showprogressbar=False)
    durasi = time.time() - mulai

    hasil[nama] = {
        "nilaiterbaik": study.bestvalue,
        "waktu": durasi
    }
    print(f"{nama}: terbaik={study.bestvalue:.4f}, waktu={durasi:.1f}s")

from optuna.pruners import MedianPruner, HyperbandPruner

Median Pruner - prune jika di bawah median trial sebelumnya
study = optuna.createstudy(
    direction="maximize",
    pruner=MedianPruner(nstartuptrials=5, nwarmupsteps=10)
)

Hyperband Pruner - early stopping agresif
study = optuna.createstudy(

    direction="maximize",
    pruner=HyperbandPruner(minresource=1, maxresource=100, reductionfactor=3)
)

import torch
import torch.nn as nn

def fungsiobjektif(trial):

    lr = trial.suggestfloat("lr", 1e-5, 1e-1, log=True)
    jumlahunit = trial.suggestint("jumlahunit", 32, 256)

    dropout = trial.suggestfloat("dropout", 0.1, 0.5)

    model = nn.Sequential(
        nn.Linear(30, jumlahunit),

        nn.ReLU(),
        nn.Dropout(dropout),
        nn.Linear(jumlahunit, jumlahunit),

        nn.ReLU(),
        nn.Dropout(dropout),
        nn.Linear(jumlahunit, 1),
        nn.Sigmoid()
    )

    optimizer = torch.optim.Adam(model.parameters(), lr=lr)
    criterion = nn.BCELoss()

    for epoch in range(100):
        model.train()
        for batchX, batchy in trainloader:

            optimizer.zerograd()
            output = model(batchX).squeeze()

            loss = criterion(output, batchy)
            loss.backward()
            optimizer.step()

        # Evaluasi dan laporkan untuk pruning
        model.eval()
        with torch.nograd():

            valoutput = model(Xvaltensor).squeeze()
            valacc = ((valoutput > 0.5) == yvaltensor).float().mean().item()

        # Laporkan nilai intermediate
        trial.report(valacc, epoch)


        # Cek apakah trial harus di-prune
        if trial.shouldprune():
            raise optuna.TrialPruned()

    return valacc


study = optuna.createstudy(
    direction="maximize",
    pruner=HyperbandPruner(minresource=5, maxresource=100)
)
study.optimize(fungsiobjektif, ntrials=50)

def multiobjektif(trial):
    nestimators = trial.suggestint("nestimators", 10, 500)
    maxdepth = trial.suggestint("maxdepth", 2, 20)


    clf = RandomForestClassifier(
        nestimators=nestimators,

        maxdepth=maxdepth,

        randomstate=42
    )

    # Objektif 1: Akurasi
    skor = crossvalscore(clf, X, y, cv=3, scoring="accuracy")
    akurasi = skor.mean()

    # Objektif 2: Waktu training (semakin rendah semakin baik)
    import time
    mulai = time.time()
    clf.fit(X, y)
    waktutraining = time.time() - mulai


    return akurasi, waktutraining

study = optuna.createstudy(

    directions=["maximize", "minimize"],
    studyname="multi-objektif-rf"
)
study.optimize(multiobjektif, ntrials=100)

Dapatkan Pareto front
trialpareto = study.besttrials
print(f"Jumlah solusi Pareto-optimal: {len(trialpareto)}")

for trial in trialpareto:
    print(f"  Akurasi: {trial.values[0]:.4f}, Waktu: {trial.values[1]:.4f}s")

import xgboost as xgb

def xgbobjektif(trial):

    params = {
        "objective": "binary:logistic",
        "evalmetric": "logloss",
        "nestimators": trial.suggestint("nestimators", 100, 1000),

        "maxdepth": trial.suggestint("maxdepth", 3, 12),
        "learningrate": trial.suggestfloat("learningrate", 1e-3, 0.3, log=True),

        "subsample": trial.suggestfloat("subsample", 0.5, 1.0),
        "colsamplebytree": trial.suggestfloat("colsamplebytree", 0.5, 1.0),

        "regalpha": trial.suggestfloat("regalpha", 1e-8, 10.0, log=True),
        "reglambda": trial.suggestfloat("reglambda", 1e-8, 10.0, log=True),

        "minchildweight": trial.suggestint("minchildweight", 1, 10),
    }

    model = xgb.XGBClassifier(params, randomstate=42, uselabelencoder=False)

    skor = crossvalscore(model, X, y, cv=5, scoring="accuracy")

    return skor.mean()

study = optuna.createstudy(direction="maximize")
study.optimize(xgbobjektif, ntrials=100)

Integrasi PyTorch Lightning

from optuna.integration import PyTorchLightningPruningCallback

def plobjektif(trial):

    lr = trial.suggestfloat("lr", 1e-5, 1e-2, log=True)
    jumlahlayer = trial.suggestint("jumlahlayer", 1, 4)

    ukuranhidden = trial.suggestint("ukuranhidden", 32, 256)

    model = ModelLightning(
        jumlahlayer=jumlahlayer,
        ukuranhidden=ukuranhidden,
        lr=lr
    )

    trainer = pl.Trainer(
        maxepochs=50,

        callbacks=[PyTorchLightningPruningCallback(trial, monitor="valacc")],
        enableprogressbar=False,
    )

    trainer.fit(model, trainloader, valloader)
    return trainer.callbackmetrics["valacc"].item()

Visualisasi dan Dashboard

from optuna.visualization import (
    plotoptimizationhistory,
    plotparamimportances,
    plotparallelcoordinate,
    plotslice,

    plotcontour,
)

Riwayat optimasi
fig = plotoptimizationhistory(study)
fig.show()

Pentingnya parameter
fig = plotparamimportances(study)
fig.show()

Plot koordinat paralel
fig = plotparallelcoordinate(study)
fig.show()

Plot slice - efek parameter individual
fig = plotslice(study)

fig.show()

Plot kontur - interaksi antar parameter
fig = plotcontour(study, params=["lr", "jumlahlayer"])

fig.show()

# Jalankan dashboard web
pip install optuna-dashboard
optuna-dashboard sqlite:///optunastudy.db --port 8080

# Simpan study ke database untuk dashboard
study = optuna.createstudy(

    studyname="eksperimen-saya",
    storage="sqlite:///optunastudy.db",

    direction="maximize",
    loadifexists=True

)

# Gunakan PostgreSQL untuk optimasi terdistribusi
storage = "postgresql://user:password@localhost:5432/optunadb"

Worker 1
study = optuna.createstudy(

    studyname="study-terdistribusi",
    storage=storage,
    direction="maximize",
    loadifexists=True
)
study.optimize(fungsiobjektif, ntrials=50)

Worker 2 (jalankan di mesin lain)
study = optuna.loadstudy(

    studyname="study-terdistribusi",
    storage=storage
)
study.optimize(fungsiobjektif, ntrials=50)

Custom Callback

def callbacklogging(study, trial):
    if study.besttrial.number == trial.number:
        print(f"Trial terbaik baru {trial.number}: nilai={trial.value:.4f}")
        print(f"  Parameter: {trial.params}")

study.optimize(fungsiobjektif, ntrials=100, callbacks=[callback_logging])

Best Practices

Kesimpulan

Optuna menyediakan framework yang powerful dan fleksibel untuk optimasi hyperparameter. Sampler yang efisien, strategi pruning, dan dukungan multi-objektif menjadikannya pilihan utama bagi praktisi ML. Dikombinasikan dengan alat visualisasi dan kemampuan terdistribusi, Optuna dapat mempercepat workflow pengembangan model Anda secara signifikan.