dflow_2b / dflow /text_encoder.py
data-archetype's picture
dflow_2b 0.2
6daffc2
Raw History Blame Contribute Delete
22.7 kB
"""Dense T5Gemma2-270M text encoder, layers 1-12, with three hidden-state taps.
The bundled ``text_encoder/`` folder holds the tokenizer and the used subset
of Google's ``t5gemma-2-270m-270m`` encoder: the token embeddings and the first
12 layers (the model reads hidden states after layers 2, 6 and 12, so later
layers never run). Each layer is a Gemma 3 block with bidirectional attention:
- embeddings scaled by ``sqrt(640)``; the end-of-image id uses its own vector;
- Gemma RMSNorm ``x * rsqrt(mean(x^2) + eps) * (1 + w)`` in FP32, four per layer
(before and after attention and the MLP);
- 4 query heads and 1 key/value head of width 256, per-head RMSNorm on queries
and keys, then RoPE; logits scaled by ``256 ** -0.5``;
- sliding layers see keys within ``-256 <= key - query <= 255`` positions
(window 512); layers 6 and 12 see every token; each kind has its own RoPE;
- GeGLU MLP (tanh GELU), width 2048.
Precision: the residual stream stays FP32 (Gemma has massive-activation
channels); matrix multiplies and attention run under CUDA bf16 autocast; the
taps are FP32.
Prompt emphasis may add a per-key logit bias and a per-token value scale in
every layer's attention; both are explicit tensor inputs.
"""
from __future__ import annotations
import json
import math
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from enum import StrEnum
from pathlib import Path
import torch
import torch.nn.functional as F
from safetensors.torch import load_file
from torch import Tensor, nn
__all__ = [
"COMPUTE_DTYPE",
"ENCODER_CONFIG",
"EncoderTaps",
"EncoderWeighting",
"LayerKind",
"T5Gemma2Encoder",
"T5Gemma2EncoderConfig",
]
# Matrix-multiply and attention dtype (autocast); the residual stream is FP32.
COMPUTE_DTYPE = torch.bfloat16
CONFIG_FILENAME = "config.json"
WEIGHTS_FILENAME = "model.safetensors"
# Stored dtype of every bundled tensor (the source checkpoint's dtype).
STORAGE_DTYPE = torch.bfloat16
class LayerKind(StrEnum):
"""Attention span of one encoder layer."""
SLIDING = "sliding_attention"
FULL = "full_attention"
class RopeType(StrEnum):
"""RoPE frequency schedules of the two layer kinds."""
DEFAULT = "default"
LINEAR = "linear"
@dataclass(frozen=True)
class RopeSettings:
"""RoPE frequencies of one layer kind.
Attributes:
theta: Base period.
rope_type: ``DEFAULT``, or ``LINEAR`` (frequencies divided by ``factor``).
factor: Linear scaling factor (1 for ``DEFAULT``).
"""
theta: float
rope_type: RopeType
factor: float
@dataclass(frozen=True)
class T5Gemma2EncoderConfig:
"""Architecture of the bundled encoder subset (``text_encoder/config.json``)."""
vocab_size: int
hidden_size: int
intermediate_size: int
num_attention_heads: int
num_key_value_heads: int
head_dim: int
query_pre_attn_scalar: float
rms_norm_eps: float
sliding_window: int
layer_types: tuple[LayerKind, ...]
sliding_rope: RopeSettings
full_rope: RopeSettings
embed_scale: float
eoi_token_index: int
tap_layers: tuple[int, int, int]
@property
def num_layers(self) -> int:
"""Return the number of bundled layers."""
return len(self.layer_types)
def rope(self, kind: LayerKind) -> RopeSettings:
"""Return the RoPE settings of ``kind``."""
match kind:
case LayerKind.SLIDING:
return self.sliding_rope
case LayerKind.FULL:
return self.full_rope
case _ as unreachable:
raise ValueError(f"Unsupported layer kind: {unreachable}")
def to_json_dict(self) -> dict[str, object]:
"""Return the JSON form written to ``text_encoder/config.json``."""
def rope(settings: RopeSettings) -> dict[str, object]:
return {
"theta": settings.theta,
"rope_type": settings.rope_type.value,
"factor": settings.factor,
}
return {
"vocab_size": self.vocab_size,
"hidden_size": self.hidden_size,
"intermediate_size": self.intermediate_size,
"num_attention_heads": self.num_attention_heads,
"num_key_value_heads": self.num_key_value_heads,
"head_dim": self.head_dim,
"query_pre_attn_scalar": self.query_pre_attn_scalar,
"rms_norm_eps": self.rms_norm_eps,
"sliding_window": self.sliding_window,
"layer_types": [kind.value for kind in self.layer_types],
"sliding_rope": rope(self.sliding_rope),
"full_rope": rope(self.full_rope),
"embed_scale": self.embed_scale,
"eoi_token_index": self.eoi_token_index,
"tap_layers": list(self.tap_layers),
}
# The used subset of google/t5gemma-2-270m-270m: embeddings and layers 1-12.
ENCODER_CONFIG = T5Gemma2EncoderConfig(
vocab_size=262_144,
hidden_size=640,
intermediate_size=2048,
num_attention_heads=4,
num_key_value_heads=1,
head_dim=256,
query_pre_attn_scalar=256.0,
rms_norm_eps=1e-6,
sliding_window=512,
layer_types=(
*(LayerKind.SLIDING,) * 5,
LayerKind.FULL,
*(LayerKind.SLIDING,) * 5,
LayerKind.FULL,
),
sliding_rope=RopeSettings(theta=10_000.0, rope_type=RopeType.DEFAULT, factor=1.0),
full_rope=RopeSettings(theta=1_000_000.0, rope_type=RopeType.LINEAR, factor=8.0),
embed_scale=640**0.5,
eoi_token_index=256_000,
tap_layers=(2, 6, 12),
)
def _check_config_file(path: Path) -> None:
"""Raise unless ``path`` holds exactly ``ENCODER_CONFIG`` (plus provenance).
Raises:
ValueError: On any architecture mismatch.
"""
raw = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(raw, dict):
raise ValueError(f"{path} must hold a JSON object.")
expected = ENCODER_CONFIG.to_json_dict()
mismatched = [key for key, value in expected.items() if raw.get(key) != value]
if mismatched:
raise ValueError(
f"{path} does not describe the bundled T5Gemma2 encoder subset; "
f"mismatched keys: {mismatched}."
)
def _rope_inv_freq(settings: RopeSettings, head_dim: int) -> Tensor:
"""Return FP32 RoPE inverse frequencies ``[head_dim // 2]``, computed on the
CPU (as the reference) so every device sees the same values."""
exponents = torch.arange(0, head_dim, 2, dtype=torch.int64, device="cpu").to(
torch.float32
)
inv_freq = 1.0 / (settings.theta ** (exponents / float(head_dim)))
match settings.rope_type:
case RopeType.DEFAULT:
return inv_freq
case RopeType.LINEAR:
return inv_freq / settings.factor
case _ as unreachable:
raise ValueError(f"Unsupported RoPE type: {unreachable}")
def _rotate_half(values: Tensor) -> Tensor:
"""Half-split rotary permutation ``[-x2, x1]``."""
half = values.shape[-1] // 2
return torch.cat((-values[..., half:], values[..., :half]), dim=-1)
class _Float32Module(nn.Module):
"""A module whose listed tensors stay FP32 across ``.to(dtype)`` moves."""
_fp32_names: tuple[str, ...] = ()
def _apply(self, fn: Callable[[Tensor], Tensor], recurse: bool = True) -> nn.Module:
"""Apply device moves without rounding the FP32 tensors."""
kept = {
name: tensor.detach().to(dtype=torch.float32)
for name, tensor in self._named_fp32()
}
super()._apply(fn, recurse=recurse)
for name, tensor in self._named_fp32():
tensor.data = kept[name].to(device=tensor.device)
return self
def _named_fp32(self) -> list[tuple[str, Tensor]]:
"""Return the FP32 tensors by name."""
named: dict[str, Tensor] = dict(self.named_parameters(recurse=False))
named.update(dict(self.named_buffers(recurse=False)))
return [(name, named[name]) for name in self._fp32_names]
class GemmaRMSNorm(_Float32Module):
"""Gemma RMSNorm with an FP32 ``(1 + w)`` affine; returns the input dtype."""
_fp32_names = ("weight",)
def __init__(self, width: int, *, eps: float) -> None:
super().__init__()
self.weight = nn.Parameter(torch.zeros(width, dtype=torch.float32))
self.eps = eps
def forward(self, values: Tensor) -> Tensor:
"""Normalize in FP32 and return the input dtype."""
normed = values.to(dtype=torch.float32)
normed = normed * torch.rsqrt(
normed.pow(2).mean(dim=-1, keepdim=True) + self.eps
)
return (normed * (1.0 + self.weight)).to(dtype=values.dtype)
class ScaledEmbedding(_Float32Module):
"""FP32 token embeddings scaled by ``embed_scale``, with the end-of-image
token's own vector."""
_fp32_names = ("weight", "eoi_embedding")
def __init__(self, config: T5Gemma2EncoderConfig) -> None:
super().__init__()
self.weight = nn.Parameter(
torch.zeros(config.vocab_size, config.hidden_size, dtype=torch.float32)
)
self.eoi_embedding = nn.Parameter(
torch.zeros(config.hidden_size, dtype=torch.float32)
)
self.embed_scale = config.embed_scale
self.eoi_token_index = config.eoi_token_index
def forward(self, ids: Tensor) -> Tensor:
"""Embed ``[B, L]`` ids as FP32 ``[B, L, hidden]``."""
values = F.embedding(ids, self.weight) * self.embed_scale
is_eoi = (ids == self.eoi_token_index).unsqueeze(-1)
return torch.where(is_eoi, self.eoi_embedding.to(values.dtype), values)
class EncoderAttention(nn.Module):
"""Bidirectional grouped-query attention of one layer."""
def __init__(self, config: T5Gemma2EncoderConfig) -> None:
super().__init__()
width, heads = config.hidden_size, config.num_attention_heads
kv_heads, head_dim = config.num_key_value_heads, config.head_dim
self.q_proj = nn.Linear(width, heads * head_dim, bias=False)
self.k_proj = nn.Linear(width, kv_heads * head_dim, bias=False)
self.v_proj = nn.Linear(width, kv_heads * head_dim, bias=False)
self.o_proj = nn.Linear(heads * head_dim, width, bias=False)
self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps)
self.k_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps)
self.heads, self.kv_heads, self.head_dim = heads, kv_heads, head_dim
self.scale = config.query_pre_attn_scalar**-0.5
def forward(
self,
values: Tensor,
*,
cos: Tensor,
sin: Tensor,
attn_bias: Tensor | None,
value_scale: Tensor | None,
) -> Tensor:
"""Attend over ``[B, L, hidden]`` values.
Args:
values: Normalized layer input.
cos, sin: FP32 RoPE tables ``[L, 1, head_dim]`` of this layer kind.
attn_bias: Additive logit bias broadcastable to ``[B, 1, L, L]``
(window mask and emphasis key bias) in the compute dtype, or
None.
value_scale: Per-token value multiplier ``[B, L, 1, 1]``, or None.
"""
batch, length, _ = values.shape
q = self.q_proj(values).view(batch, length, self.heads, self.head_dim)
k = self.k_proj(values).view(batch, length, self.kv_heads, self.head_dim)
v = self.v_proj(values).view(batch, length, self.kv_heads, self.head_dim)
q, k = self.q_norm(q), self.k_norm(k)
if value_scale is not None:
v = v * value_scale.to(dtype=v.dtype)
cos_q, sin_q = cos.to(dtype=q.dtype), sin.to(dtype=q.dtype)
q = q * cos_q + _rotate_half(q) * sin_q
k = k * cos_q + _rotate_half(k) * sin_q
groups = self.heads // self.kv_heads
out = F.scaled_dot_product_attention(
q.transpose(1, 2),
k.repeat_interleave(groups, dim=2).transpose(1, 2),
v.repeat_interleave(groups, dim=2).transpose(1, 2),
attn_mask=attn_bias,
scale=self.scale,
)
out = out.transpose(1, 2).reshape(batch, length, self.heads * self.head_dim)
return self.o_proj(out)
class EncoderMLP(nn.Module):
"""GeGLU feed-forward block (tanh GELU)."""
def __init__(self, config: T5Gemma2EncoderConfig) -> None:
super().__init__()
width, hidden = config.hidden_size, config.intermediate_size
self.gate_proj = nn.Linear(width, hidden, bias=False)
self.up_proj = nn.Linear(width, hidden, bias=False)
self.down_proj = nn.Linear(hidden, width, bias=False)
def forward(self, values: Tensor) -> Tensor:
"""Apply the gated projection."""
gate = F.gelu(self.gate_proj(values), approximate="tanh")
return self.down_proj(gate * self.up_proj(values))
class EncoderLayer(nn.Module):
"""One sandwich-normed Gemma 3 encoder block."""
def __init__(self, config: T5Gemma2EncoderConfig, kind: LayerKind) -> None:
super().__init__()
eps = config.rms_norm_eps
self.pre_self_attn_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps)
self.self_attn = EncoderAttention(config)
self.post_self_attn_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps)
self.pre_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps)
self.mlp = EncoderMLP(config)
self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps)
self.kind = kind
self._compiled: Callable[..., Tensor] | None = None
def compile_layer(self) -> None:
"""Compile this layer's forward (full graph, dynamic sequence length).
Raises:
RuntimeError: If the layer is already compiled.
"""
if self._compiled is not None:
raise RuntimeError("EncoderLayer is already compiled.")
self._compiled = torch.compile(self._forward_impl, fullgraph=True, dynamic=True)
def forward(
self,
values: Tensor,
*,
cos: Tensor,
sin: Tensor,
attn_bias: Tensor | None,
value_scale: Tensor | None,
) -> Tensor:
"""Apply the block to the FP32 residual stream ``[B, L, hidden]``."""
call = self._forward_impl if self._compiled is None else self._compiled
return call(
values, cos=cos, sin=sin, attn_bias=attn_bias, value_scale=value_scale
)
def _forward_impl(
self,
values: Tensor,
*,
cos: Tensor,
sin: Tensor,
attn_bias: Tensor | None,
value_scale: Tensor | None,
) -> Tensor:
"""Attention and MLP residual updates."""
attn = self.self_attn(
self.pre_self_attn_layernorm(values),
cos=cos,
sin=sin,
attn_bias=attn_bias,
value_scale=value_scale,
)
values = values + self.post_self_attn_layernorm(attn)
mlp = self.mlp(self.pre_feedforward_layernorm(values))
return values + self.post_feedforward_layernorm(mlp)
@dataclass(frozen=True)
class EncoderTaps:
"""FP32 hidden states ``[B, L, 640]`` after layers 2, 6 and 12."""
low: Tensor
middle: Tensor
high: Tensor
@dataclass(frozen=True)
class EncoderWeighting:
"""Prompt emphasis inside the encoder's attention.
Attributes:
key_bias: FP32 ``[B, L]`` additive logit bias on each token's key, or
None.
value_scale: FP32 ``[B, L]`` multiplier on each token's value, or None.
"""
key_bias: Tensor | None
value_scale: Tensor | None
class T5Gemma2Encoder(_Float32Module):
"""Embeddings and the first 12 T5Gemma2 encoder layers, returning three taps.
Args:
config: The architecture (``ENCODER_CONFIG``).
"""
_fp32_names = ("sliding_inv_freq", "full_inv_freq")
sliding_inv_freq: Tensor
full_inv_freq: Tensor
def __init__(self, config: T5Gemma2EncoderConfig = ENCODER_CONFIG) -> None:
super().__init__()
self.config = config
self.embed_tokens = ScaledEmbedding(config)
self.layers = nn.ModuleList(
EncoderLayer(config, kind) for kind in config.layer_types
)
for kind, name in ((LayerKind.SLIDING, "sliding"), (LayerKind.FULL, "full")):
self.register_buffer(
f"{name}_inv_freq",
_rope_inv_freq(config.rope(kind), config.head_dim),
persistent=False,
)
self._compiled = False
@classmethod
def from_directory(
cls, directory: str | Path, *, device: torch.device
) -> T5Gemma2Encoder:
"""Load the bundled weights of a ``text_encoder/`` folder onto ``device``.
Linear weights stay in their stored bf16 (the dtype autocast feeds the
matrix multiplies); embeddings and norms are FP32 copies.
Raises:
FileNotFoundError: If the weights are missing.
ValueError: If the config or any tensor name, shape or dtype differs.
"""
path = Path(directory)
_check_config_file(path / CONFIG_FILENAME)
weights_path = path / WEIGHTS_FILENAME
if not weights_path.is_file():
raise FileNotFoundError(f"No {WEIGHTS_FILENAME} in {path}.")
state = load_file(str(weights_path), device="cpu")
bad_dtypes = sorted(k for k, t in state.items() if t.dtype != STORAGE_DTYPE)
if bad_dtypes:
raise ValueError(
f"{weights_path} tensors must be {STORAGE_DTYPE}; not: {bad_dtypes}."
)
with torch.device(device):
encoder = cls(ENCODER_CONFIG)
encoder._cast_linears(STORAGE_DTYPE)
encoder.load_state_dict(state, strict=True)
return encoder.to(device=device).eval().requires_grad_(False)
def _cast_linears(self, dtype: torch.dtype) -> None:
"""Store every linear weight in ``dtype``."""
for module in self.modules():
match module:
case nn.Linear():
module.weight.data = module.weight.data.to(dtype=dtype)
case _:
pass
def _layers(self) -> list[tuple[int, EncoderLayer]]:
"""Return ``(1-based index, layer)`` pairs."""
out: list[tuple[int, EncoderLayer]] = []
for index, module in enumerate(self.layers, start=1):
match module:
case EncoderLayer():
out.append((index, module))
case _:
raise TypeError("T5Gemma2Encoder.layers must hold EncoderLayer.")
return out
def compile_layers(self) -> None:
"""Compile every layer separately (``torch.compile``, full graph,
dynamic sequence length).
Raises:
RuntimeError: If the layers are already compiled.
"""
if self._compiled:
raise RuntimeError("T5Gemma2Encoder layers are already compiled.")
for _, layer in self._layers():
layer.compile_layer()
self._compiled = True
def _rope_tables(
self, *, length: int, device: torch.device
) -> dict[LayerKind, tuple[Tensor, Tensor]]:
"""Return each layer kind's FP32 ``(cos, sin)`` ``[L, 1, head_dim]``."""
positions = torch.arange(length, device=device, dtype=torch.int64)
positions = positions.to(dtype=torch.float32).unsqueeze(1)
tables: dict[LayerKind, tuple[Tensor, Tensor]] = {}
for kind, inv_freq in (
(LayerKind.SLIDING, self.sliding_inv_freq),
(LayerKind.FULL, self.full_inv_freq),
):
freqs = positions * inv_freq.view(1, -1)
emb = torch.cat((freqs, freqs), dim=-1)
tables[kind] = (emb.cos().unsqueeze(1), emb.sin().unsqueeze(1))
return tables
def _attention_biases(
self, *, length: int, key_bias: Tensor | None, device: torch.device
) -> dict[LayerKind, Tensor | None]:
"""Return each layer kind's additive attention bias in the compute dtype.
Sliding layers mask keys outside the window when the prompt is longer
than it; the emphasis key bias is added for both kinds.
"""
key = (
None
if key_bias is None
else key_bias.to(dtype=COMPUTE_DTYPE)[:, None, None, :]
)
window = self.config.sliding_window
left, right = (window + 1) // 2 - 1, window // 2
sliding = key
if length - 1 > min(left, right):
steps = torch.arange(length, device=device)
delta = steps.view(-1, 1) - steps.view(1, -1)
outside = (delta > left) | (-delta > right)
mask = torch.zeros((length, length), device=device, dtype=COMPUTE_DTYPE)
mask = mask.masked_fill(outside, -math.inf)[None, None]
sliding = mask if key is None else mask + key
return {LayerKind.SLIDING: sliding, LayerKind.FULL: key}
def forward(
self, token_ids: Tensor, *, weighting: EncoderWeighting | None
) -> EncoderTaps:
"""Encode an unpadded ``[B, L]`` id batch on the encoder's device.
Args:
token_ids: int64 token ids, every row a full prompt (no padding).
weighting: Prompt emphasis inside attention, or None.
Returns:
The FP32 hidden states after layers 2, 6 and 12.
"""
device = token_ids.device
length = int(token_ids.shape[1])
key_bias = None if weighting is None else weighting.key_bias
value_scale = (
None
if weighting is None or weighting.value_scale is None
else weighting.value_scale[:, :, None, None]
)
biases = self._attention_biases(length=length, key_bias=key_bias, device=device)
ropes = self._rope_tables(length=length, device=device)
taps: dict[int, Tensor] = {}
with torch.autocast(device_type=device.type, dtype=COMPUTE_DTYPE):
values = self.embed_tokens(token_ids)
for index, layer in self._layers():
cos, sin = ropes[layer.kind]
values = layer(
values,
cos=cos,
sin=sin,
attn_bias=biases[layer.kind],
value_scale=value_scale,
)
if index in self.config.tap_layers:
taps[index] = values
low, middle, high = (taps[index] for index in self.config.tap_layers)
return EncoderTaps(low=low, middle=middle, high=high)