Skip to content

Shortcut Models in 2D

intermediate · Train a two-time average-velocity model with ShortcutLoss and FlowMatchingLoss as in the reference code, then sample in one, two, four or 128 jumps with FlowMapSampler.

Run it:

python examples/20-training/07-flow-maps/01-shortcut-2d/main.py
"""Shortcut Models in 2D: one-step generation by self-consistency.

A shortcut model (Frans et al., 2024) outputs the average velocity
u(x_t, t, t + d) of a jump of size d, so one call jumps from noise to data.
Training mixes two terms, as in the reference code:

    flow matching:     || u(x_t, t, t + 1/M) - (x1 - x0) ||^2
    self-consistency:  || u(x_t, t, t + 2d) - sg((u1 + u2) / 2) ||^2

where u1 and u2 are two jumps of size d from the EMA model. One eighth of each
batch goes to self-consistency, with big jumps 2d in {1, 1/2, ..., 1/64} and t
on the grid of 2d. FlowMapSampler then samples in 1, 2, 4 or 128 jumps.
"""

import os

import torch
from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn

from torchebm.datasets import TwoMoonsDataset
from torchebm.interpolants import LinearInterpolant
from torchebm.losses import FlowMatchingLoss, ShortcutLoss
from torchebm.models import TimeConditionedMLP
from torchebm.samplers import FlowMapSampler

SMOKE = os.getenv("TORCHEBM_SMOKE") == "1"
N_STEPS = 20 if SMOKE else 6000
N_GEN = 128 if SMOKE else 2000
BATCH_SIZE = 256
M = 128
LOG2_M = 7


def median_dist(samples, data):
    """Median distance from each sample to the nearest data point."""
    return torch.cdist(samples, data).min(dim=1).values.median().item()


torch.manual_seed(0)
data = TwoMoonsDataset(n_samples=4000, noise=0.05, seed=0).get_data()
model = TimeConditionedMLP(in_dim=2, prediction_type="average_velocity")
ema = AveragedModel(model, multi_avg_fn=get_ema_multi_avg_fn(0.999))
path = LinearInterpolant()
loss_fn = ShortcutLoss(target_clip=4.0, reduction="none")
fm_loss_fn = FlowMatchingLoss(reduction="none")
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

n = BATCH_SIZE // 8
k = torch.arange(LOG2_M - 1, -1, -1).repeat_interleave(n // LOG2_M)
k = torch.cat([k, k.new_zeros(n - len(k))])
d = 0.5 ** (k + 1)

for _ in range(N_STEPS):
    x1 = data[torch.randint(len(data), (BATCH_SIZE,))]
    x0 = torch.randn_like(x1)
    t = torch.floor(torch.rand(n) * 2.0**k) / 2.0**k
    x_t, _ = path(x0[:n], x1[:n], t)
    with torch.no_grad():
        u1 = ema(x_t, t + d, s=t)
        x_mid = (x_t + d[:, None] * u1).clamp(-4, 4)
        u2 = ema(x_mid, t + 2 * d, s=t + d)
    t_fm = torch.randint(M, (BATCH_SIZE - n,)) / M
    x_fm, v = path(x0[n:], x1[n:], t_fm)
    loss = torch.cat(
        [
            loss_fn(model(x_t, t + 2 * d, s=t), u1, u2),
            fm_loss_fn(model(x_fm, t_fm + 1 / M, s=t_fm), v),
        ]
    ).mean()
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    ema.update_parameters(model)

torch.manual_seed(1)
x0 = torch.randn(N_GEN, 2)
sampler = FlowMapSampler(prediction_type="average_velocity")
for n_steps in (1, 2, 4, 128):
    samples = sampler(ema.module, x0, n_steps=n_steps)
    print(f"{n_steps:3d} steps: median dist to data {median_dist(samples, data):.3f}")