"""Dense T5Gemma2-270M text encoder, layers 1-12, with three hidden-state taps. The bundled ``text_encoder/`` folder holds the tokenizer and the used subset of Google's ``t5gemma-2-270m-270m`` encoder: the token embeddings and the first 12 layers (the model reads hidden states after layers 2, 6 and 12, so later layers never run). Each layer is a Gemma 3 block with bidirectional attention: - embeddings scaled by ``sqrt(640)``; the end-of-image id uses its own vector; - Gemma RMSNorm ``x * rsqrt(mean(x^2) + eps) * (1 + w)`` in FP32, four per layer (before and after attention and the MLP); - 4 query heads and 1 key/value head of width 256, per-head RMSNorm on queries and keys, then RoPE; logits scaled by ``256 ** -0.5``; - sliding layers see keys within ``-256 <= key - query <= 255`` positions (window 512); layers 6 and 12 see every token; each kind has its own RoPE; - GeGLU MLP (tanh GELU), width 2048. Precision: the residual stream stays FP32 (Gemma has massive-activation channels); matrix multiplies and attention run under CUDA bf16 autocast; the taps are FP32. Prompt emphasis may add a per-key logit bias and a per-token value scale in every layer's attention; both are explicit tensor inputs. """ from __future__ import annotations import json import math from collections.abc import Callable, Mapping from dataclasses import dataclass from enum import StrEnum from pathlib import Path import torch import torch.nn.functional as F from safetensors.torch import load_file from torch import Tensor, nn __all__ = [ "COMPUTE_DTYPE", "ENCODER_CONFIG", "EncoderTaps", "EncoderWeighting", "LayerKind", "T5Gemma2Encoder", "T5Gemma2EncoderConfig", ] # Matrix-multiply and attention dtype (autocast); the residual stream is FP32. COMPUTE_DTYPE = torch.bfloat16 CONFIG_FILENAME = "config.json" WEIGHTS_FILENAME = "model.safetensors" # Stored dtype of every bundled tensor (the source checkpoint's dtype). STORAGE_DTYPE = torch.bfloat16 class LayerKind(StrEnum): """Attention span of one encoder layer.""" SLIDING = "sliding_attention" FULL = "full_attention" class RopeType(StrEnum): """RoPE frequency schedules of the two layer kinds.""" DEFAULT = "default" LINEAR = "linear" @dataclass(frozen=True) class RopeSettings: """RoPE frequencies of one layer kind. Attributes: theta: Base period. rope_type: ``DEFAULT``, or ``LINEAR`` (frequencies divided by ``factor``). factor: Linear scaling factor (1 for ``DEFAULT``). """ theta: float rope_type: RopeType factor: float @dataclass(frozen=True) class T5Gemma2EncoderConfig: """Architecture of the bundled encoder subset (``text_encoder/config.json``).""" vocab_size: int hidden_size: int intermediate_size: int num_attention_heads: int num_key_value_heads: int head_dim: int query_pre_attn_scalar: float rms_norm_eps: float sliding_window: int layer_types: tuple[LayerKind, ...] sliding_rope: RopeSettings full_rope: RopeSettings embed_scale: float eoi_token_index: int tap_layers: tuple[int, int, int] @property def num_layers(self) -> int: """Return the number of bundled layers.""" return len(self.layer_types) def rope(self, kind: LayerKind) -> RopeSettings: """Return the RoPE settings of ``kind``.""" match kind: case LayerKind.SLIDING: return self.sliding_rope case LayerKind.FULL: return self.full_rope case _ as unreachable: raise ValueError(f"Unsupported layer kind: {unreachable}") def to_json_dict(self) -> dict[str, object]: """Return the JSON form written to ``text_encoder/config.json``.""" def rope(settings: RopeSettings) -> dict[str, object]: return { "theta": settings.theta, "rope_type": settings.rope_type.value, "factor": settings.factor, } return { "vocab_size": self.vocab_size, "hidden_size": self.hidden_size, "intermediate_size": self.intermediate_size, "num_attention_heads": self.num_attention_heads, "num_key_value_heads": self.num_key_value_heads, "head_dim": self.head_dim, "query_pre_attn_scalar": self.query_pre_attn_scalar, "rms_norm_eps": self.rms_norm_eps, "sliding_window": self.sliding_window, "layer_types": [kind.value for kind in self.layer_types], "sliding_rope": rope(self.sliding_rope), "full_rope": rope(self.full_rope), "embed_scale": self.embed_scale, "eoi_token_index": self.eoi_token_index, "tap_layers": list(self.tap_layers), } # The used subset of google/t5gemma-2-270m-270m: embeddings and layers 1-12. ENCODER_CONFIG = T5Gemma2EncoderConfig( vocab_size=262_144, hidden_size=640, intermediate_size=2048, num_attention_heads=4, num_key_value_heads=1, head_dim=256, query_pre_attn_scalar=256.0, rms_norm_eps=1e-6, sliding_window=512, layer_types=( *(LayerKind.SLIDING,) * 5, LayerKind.FULL, *(LayerKind.SLIDING,) * 5, LayerKind.FULL, ), sliding_rope=RopeSettings(theta=10_000.0, rope_type=RopeType.DEFAULT, factor=1.0), full_rope=RopeSettings(theta=1_000_000.0, rope_type=RopeType.LINEAR, factor=8.0), embed_scale=640**0.5, eoi_token_index=256_000, tap_layers=(2, 6, 12), ) def _check_config_file(path: Path) -> None: """Raise unless ``path`` holds exactly ``ENCODER_CONFIG`` (plus provenance). Raises: ValueError: On any architecture mismatch. """ raw = json.loads(path.read_text(encoding="utf-8")) if not isinstance(raw, dict): raise ValueError(f"{path} must hold a JSON object.") expected = ENCODER_CONFIG.to_json_dict() mismatched = [key for key, value in expected.items() if raw.get(key) != value] if mismatched: raise ValueError( f"{path} does not describe the bundled T5Gemma2 encoder subset; " f"mismatched keys: {mismatched}." ) def _rope_inv_freq(settings: RopeSettings, head_dim: int) -> Tensor: """Return FP32 RoPE inverse frequencies ``[head_dim // 2]``, computed on the CPU (as the reference) so every device sees the same values.""" exponents = torch.arange(0, head_dim, 2, dtype=torch.int64, device="cpu").to( torch.float32 ) inv_freq = 1.0 / (settings.theta ** (exponents / float(head_dim))) match settings.rope_type: case RopeType.DEFAULT: return inv_freq case RopeType.LINEAR: return inv_freq / settings.factor case _ as unreachable: raise ValueError(f"Unsupported RoPE type: {unreachable}") def _rotate_half(values: Tensor) -> Tensor: """Half-split rotary permutation ``[-x2, x1]``.""" half = values.shape[-1] // 2 return torch.cat((-values[..., half:], values[..., :half]), dim=-1) class _Float32Module(nn.Module): """A module whose listed tensors stay FP32 across ``.to(dtype)`` moves.""" _fp32_names: tuple[str, ...] = () def _apply(self, fn: Callable[[Tensor], Tensor], recurse: bool = True) -> nn.Module: """Apply device moves without rounding the FP32 tensors.""" kept = { name: tensor.detach().to(dtype=torch.float32) for name, tensor in self._named_fp32() } super()._apply(fn, recurse=recurse) for name, tensor in self._named_fp32(): tensor.data = kept[name].to(device=tensor.device) return self def _named_fp32(self) -> list[tuple[str, Tensor]]: """Return the FP32 tensors by name.""" named: dict[str, Tensor] = dict(self.named_parameters(recurse=False)) named.update(dict(self.named_buffers(recurse=False))) return [(name, named[name]) for name in self._fp32_names] class GemmaRMSNorm(_Float32Module): """Gemma RMSNorm with an FP32 ``(1 + w)`` affine; returns the input dtype.""" _fp32_names = ("weight",) def __init__(self, width: int, *, eps: float) -> None: super().__init__() self.weight = nn.Parameter(torch.zeros(width, dtype=torch.float32)) self.eps = eps def forward(self, values: Tensor) -> Tensor: """Normalize in FP32 and return the input dtype.""" normed = values.to(dtype=torch.float32) normed = normed * torch.rsqrt( normed.pow(2).mean(dim=-1, keepdim=True) + self.eps ) return (normed * (1.0 + self.weight)).to(dtype=values.dtype) class ScaledEmbedding(_Float32Module): """FP32 token embeddings scaled by ``embed_scale``, with the end-of-image token's own vector.""" _fp32_names = ("weight", "eoi_embedding") def __init__(self, config: T5Gemma2EncoderConfig) -> None: super().__init__() self.weight = nn.Parameter( torch.zeros(config.vocab_size, config.hidden_size, dtype=torch.float32) ) self.eoi_embedding = nn.Parameter( torch.zeros(config.hidden_size, dtype=torch.float32) ) self.embed_scale = config.embed_scale self.eoi_token_index = config.eoi_token_index def forward(self, ids: Tensor) -> Tensor: """Embed ``[B, L]`` ids as FP32 ``[B, L, hidden]``.""" values = F.embedding(ids, self.weight) * self.embed_scale is_eoi = (ids == self.eoi_token_index).unsqueeze(-1) return torch.where(is_eoi, self.eoi_embedding.to(values.dtype), values) class EncoderAttention(nn.Module): """Bidirectional grouped-query attention of one layer.""" def __init__(self, config: T5Gemma2EncoderConfig) -> None: super().__init__() width, heads = config.hidden_size, config.num_attention_heads kv_heads, head_dim = config.num_key_value_heads, config.head_dim self.q_proj = nn.Linear(width, heads * head_dim, bias=False) self.k_proj = nn.Linear(width, kv_heads * head_dim, bias=False) self.v_proj = nn.Linear(width, kv_heads * head_dim, bias=False) self.o_proj = nn.Linear(heads * head_dim, width, bias=False) self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps) self.k_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps) self.heads, self.kv_heads, self.head_dim = heads, kv_heads, head_dim self.scale = config.query_pre_attn_scalar**-0.5 def forward( self, values: Tensor, *, cos: Tensor, sin: Tensor, attn_bias: Tensor | None, value_scale: Tensor | None, ) -> Tensor: """Attend over ``[B, L, hidden]`` values. Args: values: Normalized layer input. cos, sin: FP32 RoPE tables ``[L, 1, head_dim]`` of this layer kind. attn_bias: Additive logit bias broadcastable to ``[B, 1, L, L]`` (window mask and emphasis key bias) in the compute dtype, or None. value_scale: Per-token value multiplier ``[B, L, 1, 1]``, or None. """ batch, length, _ = values.shape q = self.q_proj(values).view(batch, length, self.heads, self.head_dim) k = self.k_proj(values).view(batch, length, self.kv_heads, self.head_dim) v = self.v_proj(values).view(batch, length, self.kv_heads, self.head_dim) q, k = self.q_norm(q), self.k_norm(k) if value_scale is not None: v = v * value_scale.to(dtype=v.dtype) cos_q, sin_q = cos.to(dtype=q.dtype), sin.to(dtype=q.dtype) q = q * cos_q + _rotate_half(q) * sin_q k = k * cos_q + _rotate_half(k) * sin_q groups = self.heads // self.kv_heads out = F.scaled_dot_product_attention( q.transpose(1, 2), k.repeat_interleave(groups, dim=2).transpose(1, 2), v.repeat_interleave(groups, dim=2).transpose(1, 2), attn_mask=attn_bias, scale=self.scale, ) out = out.transpose(1, 2).reshape(batch, length, self.heads * self.head_dim) return self.o_proj(out) class EncoderMLP(nn.Module): """GeGLU feed-forward block (tanh GELU).""" def __init__(self, config: T5Gemma2EncoderConfig) -> None: super().__init__() width, hidden = config.hidden_size, config.intermediate_size self.gate_proj = nn.Linear(width, hidden, bias=False) self.up_proj = nn.Linear(width, hidden, bias=False) self.down_proj = nn.Linear(hidden, width, bias=False) def forward(self, values: Tensor) -> Tensor: """Apply the gated projection.""" gate = F.gelu(self.gate_proj(values), approximate="tanh") return self.down_proj(gate * self.up_proj(values)) class EncoderLayer(nn.Module): """One sandwich-normed Gemma 3 encoder block.""" def __init__(self, config: T5Gemma2EncoderConfig, kind: LayerKind) -> None: super().__init__() eps = config.rms_norm_eps self.pre_self_attn_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) self.self_attn = EncoderAttention(config) self.post_self_attn_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) self.pre_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) self.mlp = EncoderMLP(config) self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) self.kind = kind self._compiled: Callable[..., Tensor] | None = None def compile_layer(self) -> None: """Compile this layer's forward (full graph, dynamic sequence length). Raises: RuntimeError: If the layer is already compiled. """ if self._compiled is not None: raise RuntimeError("EncoderLayer is already compiled.") self._compiled = torch.compile(self._forward_impl, fullgraph=True, dynamic=True) def forward( self, values: Tensor, *, cos: Tensor, sin: Tensor, attn_bias: Tensor | None, value_scale: Tensor | None, ) -> Tensor: """Apply the block to the FP32 residual stream ``[B, L, hidden]``.""" call = self._forward_impl if self._compiled is None else self._compiled return call( values, cos=cos, sin=sin, attn_bias=attn_bias, value_scale=value_scale ) def _forward_impl( self, values: Tensor, *, cos: Tensor, sin: Tensor, attn_bias: Tensor | None, value_scale: Tensor | None, ) -> Tensor: """Attention and MLP residual updates.""" attn = self.self_attn( self.pre_self_attn_layernorm(values), cos=cos, sin=sin, attn_bias=attn_bias, value_scale=value_scale, ) values = values + self.post_self_attn_layernorm(attn) mlp = self.mlp(self.pre_feedforward_layernorm(values)) return values + self.post_feedforward_layernorm(mlp) @dataclass(frozen=True) class EncoderTaps: """FP32 hidden states ``[B, L, 640]`` after layers 2, 6 and 12.""" low: Tensor middle: Tensor high: Tensor @dataclass(frozen=True) class EncoderWeighting: """Prompt emphasis inside the encoder's attention. Attributes: key_bias: FP32 ``[B, L]`` additive logit bias on each token's key, or None. value_scale: FP32 ``[B, L]`` multiplier on each token's value, or None. """ key_bias: Tensor | None value_scale: Tensor | None class T5Gemma2Encoder(_Float32Module): """Embeddings and the first 12 T5Gemma2 encoder layers, returning three taps. Args: config: The architecture (``ENCODER_CONFIG``). """ _fp32_names = ("sliding_inv_freq", "full_inv_freq") sliding_inv_freq: Tensor full_inv_freq: Tensor def __init__(self, config: T5Gemma2EncoderConfig = ENCODER_CONFIG) -> None: super().__init__() self.config = config self.embed_tokens = ScaledEmbedding(config) self.layers = nn.ModuleList( EncoderLayer(config, kind) for kind in config.layer_types ) for kind, name in ((LayerKind.SLIDING, "sliding"), (LayerKind.FULL, "full")): self.register_buffer( f"{name}_inv_freq", _rope_inv_freq(config.rope(kind), config.head_dim), persistent=False, ) self._compiled = False @classmethod def from_directory( cls, directory: str | Path, *, device: torch.device ) -> T5Gemma2Encoder: """Load the bundled weights of a ``text_encoder/`` folder onto ``device``. Linear weights stay in their stored bf16 (the dtype autocast feeds the matrix multiplies); embeddings and norms are FP32 copies. Raises: FileNotFoundError: If the weights are missing. ValueError: If the config or any tensor name, shape or dtype differs. """ path = Path(directory) _check_config_file(path / CONFIG_FILENAME) weights_path = path / WEIGHTS_FILENAME if not weights_path.is_file(): raise FileNotFoundError(f"No {WEIGHTS_FILENAME} in {path}.") state = load_file(str(weights_path), device="cpu") bad_dtypes = sorted(k for k, t in state.items() if t.dtype != STORAGE_DTYPE) if bad_dtypes: raise ValueError( f"{weights_path} tensors must be {STORAGE_DTYPE}; not: {bad_dtypes}." ) with torch.device(device): encoder = cls(ENCODER_CONFIG) encoder._cast_linears(STORAGE_DTYPE) encoder.load_state_dict(state, strict=True) return encoder.to(device=device).eval().requires_grad_(False) def _cast_linears(self, dtype: torch.dtype) -> None: """Store every linear weight in ``dtype``.""" for module in self.modules(): match module: case nn.Linear(): module.weight.data = module.weight.data.to(dtype=dtype) case _: pass def _layers(self) -> list[tuple[int, EncoderLayer]]: """Return ``(1-based index, layer)`` pairs.""" out: list[tuple[int, EncoderLayer]] = [] for index, module in enumerate(self.layers, start=1): match module: case EncoderLayer(): out.append((index, module)) case _: raise TypeError("T5Gemma2Encoder.layers must hold EncoderLayer.") return out def compile_layers(self) -> None: """Compile every layer separately (``torch.compile``, full graph, dynamic sequence length). Raises: RuntimeError: If the layers are already compiled. """ if self._compiled: raise RuntimeError("T5Gemma2Encoder layers are already compiled.") for _, layer in self._layers(): layer.compile_layer() self._compiled = True def _rope_tables( self, *, length: int, device: torch.device ) -> dict[LayerKind, tuple[Tensor, Tensor]]: """Return each layer kind's FP32 ``(cos, sin)`` ``[L, 1, head_dim]``.""" positions = torch.arange(length, device=device, dtype=torch.int64) positions = positions.to(dtype=torch.float32).unsqueeze(1) tables: dict[LayerKind, tuple[Tensor, Tensor]] = {} for kind, inv_freq in ( (LayerKind.SLIDING, self.sliding_inv_freq), (LayerKind.FULL, self.full_inv_freq), ): freqs = positions * inv_freq.view(1, -1) emb = torch.cat((freqs, freqs), dim=-1) tables[kind] = (emb.cos().unsqueeze(1), emb.sin().unsqueeze(1)) return tables def _attention_biases( self, *, length: int, key_bias: Tensor | None, device: torch.device ) -> dict[LayerKind, Tensor | None]: """Return each layer kind's additive attention bias in the compute dtype. Sliding layers mask keys outside the window when the prompt is longer than it; the emphasis key bias is added for both kinds. """ key = ( None if key_bias is None else key_bias.to(dtype=COMPUTE_DTYPE)[:, None, None, :] ) window = self.config.sliding_window left, right = (window + 1) // 2 - 1, window // 2 sliding = key if length - 1 > min(left, right): steps = torch.arange(length, device=device) delta = steps.view(-1, 1) - steps.view(1, -1) outside = (delta > left) | (-delta > right) mask = torch.zeros((length, length), device=device, dtype=COMPUTE_DTYPE) mask = mask.masked_fill(outside, -math.inf)[None, None] sliding = mask if key is None else mask + key return {LayerKind.SLIDING: sliding, LayerKind.FULL: key} def forward( self, token_ids: Tensor, *, weighting: EncoderWeighting | None ) -> EncoderTaps: """Encode an unpadded ``[B, L]`` id batch on the encoder's device. Args: token_ids: int64 token ids, every row a full prompt (no padding). weighting: Prompt emphasis inside attention, or None. Returns: The FP32 hidden states after layers 2, 6 and 12. """ device = token_ids.device length = int(token_ids.shape[1]) key_bias = None if weighting is None else weighting.key_bias value_scale = ( None if weighting is None or weighting.value_scale is None else weighting.value_scale[:, :, None, None] ) biases = self._attention_biases(length=length, key_bias=key_bias, device=device) ropes = self._rope_tables(length=length, device=device) taps: dict[int, Tensor] = {} with torch.autocast(device_type=device.type, dtype=COMPUTE_DTYPE): values = self.embed_tokens(token_ids) for index, layer in self._layers(): cos, sin = ropes[layer.kind] values = layer( values, cos=cos, sin=sin, attn_bias=biases[layer.kind], value_scale=value_scale, ) if index in self.config.tap_layers: taps[index] = values low, middle, high = (taps[index] for index in self.config.tap_layers) return EncoderTaps(low=low, middle=middle, high=high)