Skip to content

Energy Matching: the paper's 2D suite

advanced · Reproduce the paper's toy experiments end to end: 8 Gaussians transported onto two moons, with trajectories, sample evolution, LID estimation, and repulsive diverse sampling.

Run it:

python examples/20-training/04-energy-matching/02-energy-matching-paper-2d/main.py

Reproduces the 2D experiments of Energy Matching (arXiv:2504.10612) on the paper's own toy datasets: a single time-independent potential \(V(x)\) transports an 8-Gaussian ring onto two moons, then samples its Boltzmann density near the data. main.py trains with live terminal progress and prints the quantitative checks; plot.py renders the figures.

plot.py renders four figures locally (they are not committed):

  • Sample evolution along the SDE sweep: deterministic transport below \(\tau^*\), Langevin ramp after.
  • The two-regime schedule: temperature \(\epsilon(t)\) and the flow-loss time gate \(w(t)\).
  • Local intrinsic dimension: the Hessian of \(V\) at data points has one flat (tangent) and one stiff (normal) direction, so \(\mathrm{LID} = d - \mathrm{rank}(\nabla^2 V) \approx 1\) on the moons.
  • Diverse generation: 64 Langevin chains started from a single point, without and with the paper's repulsive interaction energy \(W\).
"""Energy Matching paper suite: 8 Gaussians -> Two Moons with diagnostics.

Reproduces the 2D experiments of arXiv:2504.10612 on the paper's own toy
datasets: an 8-Gaussian ring is transported onto two moons by one
time-independent potential V(x), trained with the two-phase EM recipe
(OT flow warm-up, then contrastive sharpening). Beyond generation quality,
this covers the paper's toy diagnostics: local intrinsic dimension from the
Hessian of V (their Sec. 3.3) and diverse sampling with the repulsive
interaction energy W, which fans 64 chains started from one data point out
along the manifold (the paper's inverse-design mechanism). Run plot.py
afterwards for the full figure suite.

Training calls the model for (V(x), -grad V(x)) at once; afterwards it
switches to its energy view, which the samplers differentiate, the Hessian
diagnostics read directly and InteractionModel wraps. Generation is one SDE
sweep t: 0 -> 1 started from the source distribution.

A GPU is recommended (~8 minutes end to end; CPU is much slower).
"""

import math
import os

import torch
from torch import nn
from torch.func import hessian, vmap

from torchebm.core import BaseModel, TemperatureScheduler
from torchebm.couplings import SinkhornCoupling
from torchebm.datasets import GaussianMixtureDataset, TwoMoonsDataset
from torchebm.interpolants import LinearInterpolant
from torchebm.losses import EnergyMatchingLoss
from torchebm.models import InteractionModel
from torchebm.samplers import LangevinDynamics


class MLP(BaseModel):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.net = nn.Sequential(
            nn.Linear(2, 256), nn.SiLU(),
            nn.Linear(256, 256), nn.SiLU(),
            nn.Linear(256, 256), nn.SiLU(),
            nn.Linear(256, 1),
        )

    def forward(self, x):
        return self.net(x).squeeze(-1)


def progress(label, step, total, loss):
    """Inline terminal progress bar (stdlib only)."""
    if step % 100 and step != total - 1:
        return
    bar = "=" * int(30 * (step + 1) / total)
    end = "\n" if step == total - 1 else ""
    print(
        f"\r{label} [{bar:<30}] {step + 1:5d}/{total}  loss {loss.item():.4f}",
        end=end, flush=True,
    )


def moon_distance(p):
    """Distance to the nearest point of the two analytic moon arcs."""
    dists = []
    for cx, cy, upper in ((0.0, 0.0, True), (1.0, 0.5, False)):
        v = p - torch.tensor([cx, cy], device=p.device)
        ang = torch.atan2(v[:, 1], v[:, 0])
        ang = ang.clamp(0.0, torch.pi) if upper else ang.clamp(-torch.pi, 0.0)
        arc = torch.stack([ang.cos() + cx, ang.sin() + cy], dim=-1)
        dists.append((p - arc).norm(dim=1))
    return torch.stack(dists, dim=1).min(dim=1)


SMOKE = os.getenv("TORCHEBM_SMOKE") == "1"
FLOW_STEPS, JOINT_STEPS = (20, 5) if SMOKE else (20000, 2000)
LANGEVIN_STEPS = 10 if SMOKE else 200
SWEEP_STEPS = 20 if SMOKE else 200
EPS_MAX, TAU_STAR, DT = 0.15, 0.8, 0.01

device = "cuda" if torch.cuda.is_available() else "cpu"

source = GaussianMixtureDataset(
    n_samples=8000, n_components=8, std=0.1, radius=2.5, seed=0, device=device
).get_data()
target = TwoMoonsDataset(n_samples=8000, noise=0.05, seed=1, device=device).get_data()

model = MLP(prediction_type=("energy", "score")).to(device)
coupling = SinkhornCoupling(reg=0.01)
path = LinearInterpolant()
loss_fn = EnergyMatchingLoss(lambda_cd=2.0, cd_trim_fraction=0.0, cd_clamp=None)
langevin_sweep = LangevinDynamics(
    step_size=DT,
    noise_scale=TemperatureScheduler(
        epsilon_max=EPS_MAX, tau_star=TAU_STAR, n_steps=LANGEVIN_STEPS
    ),
)
langevin_hold = LangevinDynamics(step_size=DT, noise_scale=math.sqrt(EPS_MAX))
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)


def next_batch():
    i1 = torch.randint(len(target), (128,), device=device)
    i0 = torch.randint(len(source), (128,), device=device)
    return target[i1], source[i0]


def compute_loss(x1, x0, contrastive):
    x0, x1 = coupling(x0, x1)
    t = torch.rand(len(x1), device=device)
    x_t, u_t = path(x0, x1, t)
    x_t = x_t + 0.1 * torch.randn_like(x_t)
    _, score = model(x_t)
    if not contrastive:
        return loss_fn(score, u_t, t)
    half = len(x1) // 2
    negatives = torch.cat([
        langevin_sweep(model, x0[:half], n_steps=LANGEVIN_STEPS),
        langevin_hold(model, x1[half:], n_steps=LANGEVIN_STEPS),
    ])
    energy_pos, _ = model(x1)
    energy_neg, _ = model(negatives)
    return loss_fn(score, u_t, t, energy_pos, energy_neg)


for step in range(FLOW_STEPS):
    x1, x0 = next_batch()
    loss = compute_loss(x1, x0, contrastive=False)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    progress("phase 1 (OT flow)", step, FLOW_STEPS, loss)

for step in range(JOINT_STEPS):
    x1, x0 = next_batch()
    loss = compute_loss(x1, x0, contrastive=True)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    progress("phase 2 (EM)     ", step, JOINT_STEPS, loss)

model.prediction_type = "energy"

temperature = TemperatureScheduler(
    epsilon_max=EPS_MAX, tau_star=TAU_STAR, n_steps=SWEEP_STEPS, t_end=1.0
)
sampler = LangevinDynamics(step_size=DT, noise_scale=temperature)
x_init = source[torch.randperm(len(source), device=device)[:4000]]
samples = sampler(model, x_init, n_steps=SWEEP_STEPS)

near = moon_distance(samples)
outer = (near.indices == 0).float().mean()
print(
    f"transport 8 Gaussians -> moons: median distance {near.values.median():.3f}"
    f"  within 0.15: {(near.values < 0.15).float().mean():.1%}"
    f"  outer/inner split: {outer:.1%}/{1 - outer:.1%}"
)


def hessian_eigs(points):
    """Eigenvalues of the Hessian of V at `points`.

    LID = d - rank(grad^2 V): flat (small-eigenvalue) directions are tangent to
    the manifold. The rank uses a per-point relative threshold and is
    meaningful on the density ridge, where V is convex. The moons are curves,
    so the true LID is 1.
    """
    v_single = lambda p: model(p.unsqueeze(0)).squeeze(0)
    return torch.linalg.eigvalsh(vmap(hessian(v_single))(points)).detach()


eig_data = hessian_eigs(target[:512])
rank = (eig_data > 0.3 * eig_data[:, 1:].clamp(min=1e-6)).sum(dim=1)
lid_data = 2.0 - rank.float()
x_random = torch.rand(512, 2, device=device) * 4.0 - torch.tensor([1.5, 1.5], device=device)
eig_random = hessian_eigs(x_random)
print(
    f"LID at data points: mean {lid_data.mean():.2f} (true manifold dim: 1)"
    f"   stiff-eigenvalue medians: data {eig_data[:, 1].median():.1f}"
    f" vs off-manifold {eig_random[:, 1].median():.1f}"
)

x_seed = target[:1].expand(64, -1).contiguous()
samples_plain = langevin_hold(model, x_seed, n_steps=SWEEP_STEPS)
interaction_model = InteractionModel(model, sigma_w=4.0, strength=0.15)
samples_repulsive = langevin_hold(interaction_model, x_seed, n_steps=SWEEP_STEPS)


def mean_pairwise(x):
    return torch.pdist(x).mean()


print(
    f"diversity (mean pairwise distance) from one seed point:"
    f"  plain {mean_pairwise(samples_plain):.2f}"
    f"  vs  repulsive {mean_pairwise(samples_repulsive):.2f}"
)