Add grouped-query attention with parity tests against multi-head

GQA/MQA via GPTConfig.n_kv_head; the cache stores only the KV heads. Attention
also accepts per-row positions and a key padding mask, used by batched decoding.
This commit is contained in:
2026-11-07 20:25:52 -07:00
parent e8a20db1a3
commit a603d953c1
5 changed files with 241 additions and 17 deletions
+19
View File
@@ -0,0 +1,19 @@
import torch
import torch.nn as nn
from microgpt.model import GPT, GPTConfig
def make_model(cfg: GPTConfig, seed: int = 0, std: float = 0.3) -> GPT:
"""Random model with *confident* outputs.
The default init (std 0.02) gives near-uniform logits, where greedy argmax can flip on
float noise and tests would compare ties. A wider init makes distinct, stable argmaxes.
"""
torch.manual_seed(seed)
model = GPT(cfg)
with torch.no_grad():
for mod in model.modules():
if isinstance(mod, (nn.Linear, nn.Embedding)):
mod.weight.normal_(0.0, std)
return model.eval()