import torch
from diffusers.models import AutoencoderKL
from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn
from torch.utils.data import DataLoader, TensorDataset
from torchvision import datasets, transforms
from torchebm.interpolants import LinearInterpolant
from torchebm.losses import EquilibriumMatchingLoss
from torchebm.models import ClassifierFreeGuidance, dit_xl_2
from torchebm.samplers import NesterovSampler
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(256),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),
])
dataset = datasets.ImageFolder("imagenet/train", transform)
vae = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-ema").cuda()
moments = torch.empty(len(dataset), 2, 8, 32, 32, dtype=torch.float16)
with torch.no_grad():
for i, (images, _) in enumerate(DataLoader(dataset, batch_size=256, num_workers=8)):
for mirror, view in enumerate([images, images.flip(-1)]):
latent = vae.encode(view.cuda()).latent_dist
moments[256 * i : 256 * i + len(view), mirror] = torch.cat([latent.mean, latent.std], 1)
labels = torch.tensor(dataset.targets)
loader = DataLoader(TensorDataset(moments, labels), batch_size=256, shuffle=True, drop_last=True)
model = dit_xl_2(
input_size=32, in_channels=4, num_classes=1000, class_dropout_prob=0.1,
time_invariant=True,
).cuda()
optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4, weight_decay=0)
ema = AveragedModel(model, multi_avg_fn=get_ema_multi_avg_fn(0.9999))
path, loss_fn = LinearInterpolant(), EquilibriumMatchingLoss()
for epoch in range(1400):
for batch, y in loader:
mirror = torch.randint(2, (len(y),))
mean, std = batch[torch.arange(len(y)), mirror].cuda().float().chunk(2, dim=1)
x1, y = (mean + std * torch.randn_like(std)) * 0.18215, y.cuda()
x0, t = torch.randn_like(x1), torch.rand(len(x1), device="cuda")
x_t, u_t = path(x0, x1, t)
loss = loss_fn(model(x_t, y=y), u_t, t)
optimizer.zero_grad()
loss.backward()
optimizer.step()
ema.update_parameters(model)
guided = ClassifierFreeGuidance(
ema.module.eval(), guidance_scale=1.5, null_condition=1000, guide_channels=3
)
sampler = NesterovSampler(
step_size=0.0017, momentum=0.3, lookahead_only=True, prediction_type="velocity"
)
y = torch.randint(1000, (64,), device="cuda")
noise = torch.randn(64, 4, 32, 32, device="cuda")
latents = sampler(guided, noise, n_steps=249, model_kwargs={"y": y})
with torch.no_grad():
samples = vae.decode(latents / 0.18215).sample