Download dflow/text_encoder.py from data-archetype/dflow_2b: direct link, hf CLI and curl.
- Browser
- Download file 22.7 kB
-
https://huggingface.co/data-archetype/dflow_2b/resolve/main/dflow/text_encoder.py
- Command line
-
hf download hf://data-archetype/dflow_2b/dflow/text_encoder.py
-
curl -L -o text_encoder.py https://huggingface.co/data-archetype/dflow_2b/resolve/main/dflow/text_encoder.py
22.7 kB
| """Dense T5Gemma2-270M text encoder, layers 1-12, with three hidden-state taps. | |
| The bundled ``text_encoder/`` folder holds the tokenizer and the used subset | |
| of Google's ``t5gemma-2-270m-270m`` encoder: the token embeddings and the first | |
| 12 layers (the model reads hidden states after layers 2, 6 and 12, so later | |
| layers never run). Each layer is a Gemma 3 block with bidirectional attention: | |
| - embeddings scaled by ``sqrt(640)``; the end-of-image id uses its own vector; | |
| - Gemma RMSNorm ``x * rsqrt(mean(x^2) + eps) * (1 + w)`` in FP32, four per layer | |
| (before and after attention and the MLP); | |
| - 4 query heads and 1 key/value head of width 256, per-head RMSNorm on queries | |
| and keys, then RoPE; logits scaled by ``256 ** -0.5``; | |
| - sliding layers see keys within ``-256 <= key - query <= 255`` positions | |
| (window 512); layers 6 and 12 see every token; each kind has its own RoPE; | |
| - GeGLU MLP (tanh GELU), width 2048. | |
| Precision: the residual stream stays FP32 (Gemma has massive-activation | |
| channels); matrix multiplies and attention run under CUDA bf16 autocast; the | |
| taps are FP32. | |
| Prompt emphasis may add a per-key logit bias and a per-token value scale in | |
| every layer's attention; both are explicit tensor inputs. | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import math | |
| from collections.abc import Callable, Mapping | |
| from dataclasses import dataclass | |
| from enum import StrEnum | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| from torch import Tensor, nn | |
| __all__ = [ | |
| "COMPUTE_DTYPE", | |
| "ENCODER_CONFIG", | |
| "EncoderTaps", | |
| "EncoderWeighting", | |
| "LayerKind", | |
| "T5Gemma2Encoder", | |
| "T5Gemma2EncoderConfig", | |
| ] | |
| # Matrix-multiply and attention dtype (autocast); the residual stream is FP32. | |
| COMPUTE_DTYPE = torch.bfloat16 | |
| CONFIG_FILENAME = "config.json" | |
| WEIGHTS_FILENAME = "model.safetensors" | |
| # Stored dtype of every bundled tensor (the source checkpoint's dtype). | |
| STORAGE_DTYPE = torch.bfloat16 | |
| class LayerKind(StrEnum): | |
| """Attention span of one encoder layer.""" | |
| SLIDING = "sliding_attention" | |
| FULL = "full_attention" | |
| class RopeType(StrEnum): | |
| """RoPE frequency schedules of the two layer kinds.""" | |
| DEFAULT = "default" | |
| LINEAR = "linear" | |
| class RopeSettings: | |
| """RoPE frequencies of one layer kind. | |
| Attributes: | |
| theta: Base period. | |
| rope_type: ``DEFAULT``, or ``LINEAR`` (frequencies divided by ``factor``). | |
| factor: Linear scaling factor (1 for ``DEFAULT``). | |
| """ | |
| theta: float | |
| rope_type: RopeType | |
| factor: float | |
| class T5Gemma2EncoderConfig: | |
| """Architecture of the bundled encoder subset (``text_encoder/config.json``).""" | |
| vocab_size: int | |
| hidden_size: int | |
| intermediate_size: int | |
| num_attention_heads: int | |
| num_key_value_heads: int | |
| head_dim: int | |
| query_pre_attn_scalar: float | |
| rms_norm_eps: float | |
| sliding_window: int | |
| layer_types: tuple[LayerKind, ...] | |
| sliding_rope: RopeSettings | |
| full_rope: RopeSettings | |
| embed_scale: float | |
| eoi_token_index: int | |
| tap_layers: tuple[int, int, int] | |
| def num_layers(self) -> int: | |
| """Return the number of bundled layers.""" | |
| return len(self.layer_types) | |
| def rope(self, kind: LayerKind) -> RopeSettings: | |
| """Return the RoPE settings of ``kind``.""" | |
| match kind: | |
| case LayerKind.SLIDING: | |
| return self.sliding_rope | |
| case LayerKind.FULL: | |
| return self.full_rope | |
| case _ as unreachable: | |
| raise ValueError(f"Unsupported layer kind: {unreachable}") | |
| def to_json_dict(self) -> dict[str, object]: | |
| """Return the JSON form written to ``text_encoder/config.json``.""" | |
| def rope(settings: RopeSettings) -> dict[str, object]: | |
| return { | |
| "theta": settings.theta, | |
| "rope_type": settings.rope_type.value, | |
| "factor": settings.factor, | |
| } | |
| return { | |
| "vocab_size": self.vocab_size, | |
| "hidden_size": self.hidden_size, | |
| "intermediate_size": self.intermediate_size, | |
| "num_attention_heads": self.num_attention_heads, | |
| "num_key_value_heads": self.num_key_value_heads, | |
| "head_dim": self.head_dim, | |
| "query_pre_attn_scalar": self.query_pre_attn_scalar, | |
| "rms_norm_eps": self.rms_norm_eps, | |
| "sliding_window": self.sliding_window, | |
| "layer_types": [kind.value for kind in self.layer_types], | |
| "sliding_rope": rope(self.sliding_rope), | |
| "full_rope": rope(self.full_rope), | |
| "embed_scale": self.embed_scale, | |
| "eoi_token_index": self.eoi_token_index, | |
| "tap_layers": list(self.tap_layers), | |
| } | |
| # The used subset of google/t5gemma-2-270m-270m: embeddings and layers 1-12. | |
| ENCODER_CONFIG = T5Gemma2EncoderConfig( | |
| vocab_size=262_144, | |
| hidden_size=640, | |
| intermediate_size=2048, | |
| num_attention_heads=4, | |
| num_key_value_heads=1, | |
| head_dim=256, | |
| query_pre_attn_scalar=256.0, | |
| rms_norm_eps=1e-6, | |
| sliding_window=512, | |
| layer_types=( | |
| *(LayerKind.SLIDING,) * 5, | |
| LayerKind.FULL, | |
| *(LayerKind.SLIDING,) * 5, | |
| LayerKind.FULL, | |
| ), | |
| sliding_rope=RopeSettings(theta=10_000.0, rope_type=RopeType.DEFAULT, factor=1.0), | |
| full_rope=RopeSettings(theta=1_000_000.0, rope_type=RopeType.LINEAR, factor=8.0), | |
| embed_scale=640**0.5, | |
| eoi_token_index=256_000, | |
| tap_layers=(2, 6, 12), | |
| ) | |
| def _check_config_file(path: Path) -> None: | |
| """Raise unless ``path`` holds exactly ``ENCODER_CONFIG`` (plus provenance). | |
| Raises: | |
| ValueError: On any architecture mismatch. | |
| """ | |
| raw = json.loads(path.read_text(encoding="utf-8")) | |
| if not isinstance(raw, dict): | |
| raise ValueError(f"{path} must hold a JSON object.") | |
| expected = ENCODER_CONFIG.to_json_dict() | |
| mismatched = [key for key, value in expected.items() if raw.get(key) != value] | |
| if mismatched: | |
| raise ValueError( | |
| f"{path} does not describe the bundled T5Gemma2 encoder subset; " | |
| f"mismatched keys: {mismatched}." | |
| ) | |
| def _rope_inv_freq(settings: RopeSettings, head_dim: int) -> Tensor: | |
| """Return FP32 RoPE inverse frequencies ``[head_dim // 2]``, computed on the | |
| CPU (as the reference) so every device sees the same values.""" | |
| exponents = torch.arange(0, head_dim, 2, dtype=torch.int64, device="cpu").to( | |
| torch.float32 | |
| ) | |
| inv_freq = 1.0 / (settings.theta ** (exponents / float(head_dim))) | |
| match settings.rope_type: | |
| case RopeType.DEFAULT: | |
| return inv_freq | |
| case RopeType.LINEAR: | |
| return inv_freq / settings.factor | |
| case _ as unreachable: | |
| raise ValueError(f"Unsupported RoPE type: {unreachable}") | |
| def _rotate_half(values: Tensor) -> Tensor: | |
| """Half-split rotary permutation ``[-x2, x1]``.""" | |
| half = values.shape[-1] // 2 | |
| return torch.cat((-values[..., half:], values[..., :half]), dim=-1) | |
| class _Float32Module(nn.Module): | |
| """A module whose listed tensors stay FP32 across ``.to(dtype)`` moves.""" | |
| _fp32_names: tuple[str, ...] = () | |
| def _apply(self, fn: Callable[[Tensor], Tensor], recurse: bool = True) -> nn.Module: | |
| """Apply device moves without rounding the FP32 tensors.""" | |
| kept = { | |
| name: tensor.detach().to(dtype=torch.float32) | |
| for name, tensor in self._named_fp32() | |
| } | |
| super()._apply(fn, recurse=recurse) | |
| for name, tensor in self._named_fp32(): | |
| tensor.data = kept[name].to(device=tensor.device) | |
| return self | |
| def _named_fp32(self) -> list[tuple[str, Tensor]]: | |
| """Return the FP32 tensors by name.""" | |
| named: dict[str, Tensor] = dict(self.named_parameters(recurse=False)) | |
| named.update(dict(self.named_buffers(recurse=False))) | |
| return [(name, named[name]) for name in self._fp32_names] | |
| class GemmaRMSNorm(_Float32Module): | |
| """Gemma RMSNorm with an FP32 ``(1 + w)`` affine; returns the input dtype.""" | |
| _fp32_names = ("weight",) | |
| def __init__(self, width: int, *, eps: float) -> None: | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.zeros(width, dtype=torch.float32)) | |
| self.eps = eps | |
| def forward(self, values: Tensor) -> Tensor: | |
| """Normalize in FP32 and return the input dtype.""" | |
| normed = values.to(dtype=torch.float32) | |
| normed = normed * torch.rsqrt( | |
| normed.pow(2).mean(dim=-1, keepdim=True) + self.eps | |
| ) | |
| return (normed * (1.0 + self.weight)).to(dtype=values.dtype) | |
| class ScaledEmbedding(_Float32Module): | |
| """FP32 token embeddings scaled by ``embed_scale``, with the end-of-image | |
| token's own vector.""" | |
| _fp32_names = ("weight", "eoi_embedding") | |
| def __init__(self, config: T5Gemma2EncoderConfig) -> None: | |
| super().__init__() | |
| self.weight = nn.Parameter( | |
| torch.zeros(config.vocab_size, config.hidden_size, dtype=torch.float32) | |
| ) | |
| self.eoi_embedding = nn.Parameter( | |
| torch.zeros(config.hidden_size, dtype=torch.float32) | |
| ) | |
| self.embed_scale = config.embed_scale | |
| self.eoi_token_index = config.eoi_token_index | |
| def forward(self, ids: Tensor) -> Tensor: | |
| """Embed ``[B, L]`` ids as FP32 ``[B, L, hidden]``.""" | |
| values = F.embedding(ids, self.weight) * self.embed_scale | |
| is_eoi = (ids == self.eoi_token_index).unsqueeze(-1) | |
| return torch.where(is_eoi, self.eoi_embedding.to(values.dtype), values) | |
| class EncoderAttention(nn.Module): | |
| """Bidirectional grouped-query attention of one layer.""" | |
| def __init__(self, config: T5Gemma2EncoderConfig) -> None: | |
| super().__init__() | |
| width, heads = config.hidden_size, config.num_attention_heads | |
| kv_heads, head_dim = config.num_key_value_heads, config.head_dim | |
| self.q_proj = nn.Linear(width, heads * head_dim, bias=False) | |
| self.k_proj = nn.Linear(width, kv_heads * head_dim, bias=False) | |
| self.v_proj = nn.Linear(width, kv_heads * head_dim, bias=False) | |
| self.o_proj = nn.Linear(heads * head_dim, width, bias=False) | |
| self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps) | |
| self.k_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps) | |
| self.heads, self.kv_heads, self.head_dim = heads, kv_heads, head_dim | |
| self.scale = config.query_pre_attn_scalar**-0.5 | |
| def forward( | |
| self, | |
| values: Tensor, | |
| *, | |
| cos: Tensor, | |
| sin: Tensor, | |
| attn_bias: Tensor | None, | |
| value_scale: Tensor | None, | |
| ) -> Tensor: | |
| """Attend over ``[B, L, hidden]`` values. | |
| Args: | |
| values: Normalized layer input. | |
| cos, sin: FP32 RoPE tables ``[L, 1, head_dim]`` of this layer kind. | |
| attn_bias: Additive logit bias broadcastable to ``[B, 1, L, L]`` | |
| (window mask and emphasis key bias) in the compute dtype, or | |
| None. | |
| value_scale: Per-token value multiplier ``[B, L, 1, 1]``, or None. | |
| """ | |
| batch, length, _ = values.shape | |
| q = self.q_proj(values).view(batch, length, self.heads, self.head_dim) | |
| k = self.k_proj(values).view(batch, length, self.kv_heads, self.head_dim) | |
| v = self.v_proj(values).view(batch, length, self.kv_heads, self.head_dim) | |
| q, k = self.q_norm(q), self.k_norm(k) | |
| if value_scale is not None: | |
| v = v * value_scale.to(dtype=v.dtype) | |
| cos_q, sin_q = cos.to(dtype=q.dtype), sin.to(dtype=q.dtype) | |
| q = q * cos_q + _rotate_half(q) * sin_q | |
| k = k * cos_q + _rotate_half(k) * sin_q | |
| groups = self.heads // self.kv_heads | |
| out = F.scaled_dot_product_attention( | |
| q.transpose(1, 2), | |
| k.repeat_interleave(groups, dim=2).transpose(1, 2), | |
| v.repeat_interleave(groups, dim=2).transpose(1, 2), | |
| attn_mask=attn_bias, | |
| scale=self.scale, | |
| ) | |
| out = out.transpose(1, 2).reshape(batch, length, self.heads * self.head_dim) | |
| return self.o_proj(out) | |
| class EncoderMLP(nn.Module): | |
| """GeGLU feed-forward block (tanh GELU).""" | |
| def __init__(self, config: T5Gemma2EncoderConfig) -> None: | |
| super().__init__() | |
| width, hidden = config.hidden_size, config.intermediate_size | |
| self.gate_proj = nn.Linear(width, hidden, bias=False) | |
| self.up_proj = nn.Linear(width, hidden, bias=False) | |
| self.down_proj = nn.Linear(hidden, width, bias=False) | |
| def forward(self, values: Tensor) -> Tensor: | |
| """Apply the gated projection.""" | |
| gate = F.gelu(self.gate_proj(values), approximate="tanh") | |
| return self.down_proj(gate * self.up_proj(values)) | |
| class EncoderLayer(nn.Module): | |
| """One sandwich-normed Gemma 3 encoder block.""" | |
| def __init__(self, config: T5Gemma2EncoderConfig, kind: LayerKind) -> None: | |
| super().__init__() | |
| eps = config.rms_norm_eps | |
| self.pre_self_attn_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) | |
| self.self_attn = EncoderAttention(config) | |
| self.post_self_attn_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) | |
| self.pre_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) | |
| self.mlp = EncoderMLP(config) | |
| self.post_feedforward_layernorm = GemmaRMSNorm(config.hidden_size, eps=eps) | |
| self.kind = kind | |
| self._compiled: Callable[..., Tensor] | None = None | |
| def compile_layer(self) -> None: | |
| """Compile this layer's forward (full graph, dynamic sequence length). | |
| Raises: | |
| RuntimeError: If the layer is already compiled. | |
| """ | |
| if self._compiled is not None: | |
| raise RuntimeError("EncoderLayer is already compiled.") | |
| self._compiled = torch.compile(self._forward_impl, fullgraph=True, dynamic=True) | |
| def forward( | |
| self, | |
| values: Tensor, | |
| *, | |
| cos: Tensor, | |
| sin: Tensor, | |
| attn_bias: Tensor | None, | |
| value_scale: Tensor | None, | |
| ) -> Tensor: | |
| """Apply the block to the FP32 residual stream ``[B, L, hidden]``.""" | |
| call = self._forward_impl if self._compiled is None else self._compiled | |
| return call( | |
| values, cos=cos, sin=sin, attn_bias=attn_bias, value_scale=value_scale | |
| ) | |
| def _forward_impl( | |
| self, | |
| values: Tensor, | |
| *, | |
| cos: Tensor, | |
| sin: Tensor, | |
| attn_bias: Tensor | None, | |
| value_scale: Tensor | None, | |
| ) -> Tensor: | |
| """Attention and MLP residual updates.""" | |
| attn = self.self_attn( | |
| self.pre_self_attn_layernorm(values), | |
| cos=cos, | |
| sin=sin, | |
| attn_bias=attn_bias, | |
| value_scale=value_scale, | |
| ) | |
| values = values + self.post_self_attn_layernorm(attn) | |
| mlp = self.mlp(self.pre_feedforward_layernorm(values)) | |
| return values + self.post_feedforward_layernorm(mlp) | |
| class EncoderTaps: | |
| """FP32 hidden states ``[B, L, 640]`` after layers 2, 6 and 12.""" | |
| low: Tensor | |
| middle: Tensor | |
| high: Tensor | |
| class EncoderWeighting: | |
| """Prompt emphasis inside the encoder's attention. | |
| Attributes: | |
| key_bias: FP32 ``[B, L]`` additive logit bias on each token's key, or | |
| None. | |
| value_scale: FP32 ``[B, L]`` multiplier on each token's value, or None. | |
| """ | |
| key_bias: Tensor | None | |
| value_scale: Tensor | None | |
| class T5Gemma2Encoder(_Float32Module): | |
| """Embeddings and the first 12 T5Gemma2 encoder layers, returning three taps. | |
| Args: | |
| config: The architecture (``ENCODER_CONFIG``). | |
| """ | |
| _fp32_names = ("sliding_inv_freq", "full_inv_freq") | |
| sliding_inv_freq: Tensor | |
| full_inv_freq: Tensor | |
| def __init__(self, config: T5Gemma2EncoderConfig = ENCODER_CONFIG) -> None: | |
| super().__init__() | |
| self.config = config | |
| self.embed_tokens = ScaledEmbedding(config) | |
| self.layers = nn.ModuleList( | |
| EncoderLayer(config, kind) for kind in config.layer_types | |
| ) | |
| for kind, name in ((LayerKind.SLIDING, "sliding"), (LayerKind.FULL, "full")): | |
| self.register_buffer( | |
| f"{name}_inv_freq", | |
| _rope_inv_freq(config.rope(kind), config.head_dim), | |
| persistent=False, | |
| ) | |
| self._compiled = False | |
| def from_directory( | |
| cls, directory: str | Path, *, device: torch.device | |
| ) -> T5Gemma2Encoder: | |
| """Load the bundled weights of a ``text_encoder/`` folder onto ``device``. | |
| Linear weights stay in their stored bf16 (the dtype autocast feeds the | |
| matrix multiplies); embeddings and norms are FP32 copies. | |
| Raises: | |
| FileNotFoundError: If the weights are missing. | |
| ValueError: If the config or any tensor name, shape or dtype differs. | |
| """ | |
| path = Path(directory) | |
| _check_config_file(path / CONFIG_FILENAME) | |
| weights_path = path / WEIGHTS_FILENAME | |
| if not weights_path.is_file(): | |
| raise FileNotFoundError(f"No {WEIGHTS_FILENAME} in {path}.") | |
| state = load_file(str(weights_path), device="cpu") | |
| bad_dtypes = sorted(k for k, t in state.items() if t.dtype != STORAGE_DTYPE) | |
| if bad_dtypes: | |
| raise ValueError( | |
| f"{weights_path} tensors must be {STORAGE_DTYPE}; not: {bad_dtypes}." | |
| ) | |
| with torch.device(device): | |
| encoder = cls(ENCODER_CONFIG) | |
| encoder._cast_linears(STORAGE_DTYPE) | |
| encoder.load_state_dict(state, strict=True) | |
| return encoder.to(device=device).eval().requires_grad_(False) | |
| def _cast_linears(self, dtype: torch.dtype) -> None: | |
| """Store every linear weight in ``dtype``.""" | |
| for module in self.modules(): | |
| match module: | |
| case nn.Linear(): | |
| module.weight.data = module.weight.data.to(dtype=dtype) | |
| case _: | |
| pass | |
| def _layers(self) -> list[tuple[int, EncoderLayer]]: | |
| """Return ``(1-based index, layer)`` pairs.""" | |
| out: list[tuple[int, EncoderLayer]] = [] | |
| for index, module in enumerate(self.layers, start=1): | |
| match module: | |
| case EncoderLayer(): | |
| out.append((index, module)) | |
| case _: | |
| raise TypeError("T5Gemma2Encoder.layers must hold EncoderLayer.") | |
| return out | |
| def compile_layers(self) -> None: | |
| """Compile every layer separately (``torch.compile``, full graph, | |
| dynamic sequence length). | |
| Raises: | |
| RuntimeError: If the layers are already compiled. | |
| """ | |
| if self._compiled: | |
| raise RuntimeError("T5Gemma2Encoder layers are already compiled.") | |
| for _, layer in self._layers(): | |
| layer.compile_layer() | |
| self._compiled = True | |
| def _rope_tables( | |
| self, *, length: int, device: torch.device | |
| ) -> dict[LayerKind, tuple[Tensor, Tensor]]: | |
| """Return each layer kind's FP32 ``(cos, sin)`` ``[L, 1, head_dim]``.""" | |
| positions = torch.arange(length, device=device, dtype=torch.int64) | |
| positions = positions.to(dtype=torch.float32).unsqueeze(1) | |
| tables: dict[LayerKind, tuple[Tensor, Tensor]] = {} | |
| for kind, inv_freq in ( | |
| (LayerKind.SLIDING, self.sliding_inv_freq), | |
| (LayerKind.FULL, self.full_inv_freq), | |
| ): | |
| freqs = positions * inv_freq.view(1, -1) | |
| emb = torch.cat((freqs, freqs), dim=-1) | |
| tables[kind] = (emb.cos().unsqueeze(1), emb.sin().unsqueeze(1)) | |
| return tables | |
| def _attention_biases( | |
| self, *, length: int, key_bias: Tensor | None, device: torch.device | |
| ) -> dict[LayerKind, Tensor | None]: | |
| """Return each layer kind's additive attention bias in the compute dtype. | |
| Sliding layers mask keys outside the window when the prompt is longer | |
| than it; the emphasis key bias is added for both kinds. | |
| """ | |
| key = ( | |
| None | |
| if key_bias is None | |
| else key_bias.to(dtype=COMPUTE_DTYPE)[:, None, None, :] | |
| ) | |
| window = self.config.sliding_window | |
| left, right = (window + 1) // 2 - 1, window // 2 | |
| sliding = key | |
| if length - 1 > min(left, right): | |
| steps = torch.arange(length, device=device) | |
| delta = steps.view(-1, 1) - steps.view(1, -1) | |
| outside = (delta > left) | (-delta > right) | |
| mask = torch.zeros((length, length), device=device, dtype=COMPUTE_DTYPE) | |
| mask = mask.masked_fill(outside, -math.inf)[None, None] | |
| sliding = mask if key is None else mask + key | |
| return {LayerKind.SLIDING: sliding, LayerKind.FULL: key} | |
| def forward( | |
| self, token_ids: Tensor, *, weighting: EncoderWeighting | None | |
| ) -> EncoderTaps: | |
| """Encode an unpadded ``[B, L]`` id batch on the encoder's device. | |
| Args: | |
| token_ids: int64 token ids, every row a full prompt (no padding). | |
| weighting: Prompt emphasis inside attention, or None. | |
| Returns: | |
| The FP32 hidden states after layers 2, 6 and 12. | |
| """ | |
| device = token_ids.device | |
| length = int(token_ids.shape[1]) | |
| key_bias = None if weighting is None else weighting.key_bias | |
| value_scale = ( | |
| None | |
| if weighting is None or weighting.value_scale is None | |
| else weighting.value_scale[:, :, None, None] | |
| ) | |
| biases = self._attention_biases(length=length, key_bias=key_bias, device=device) | |
| ropes = self._rope_tables(length=length, device=device) | |
| taps: dict[int, Tensor] = {} | |
| with torch.autocast(device_type=device.type, dtype=COMPUTE_DTYPE): | |
| values = self.embed_tokens(token_ids) | |
| for index, layer in self._layers(): | |
| cos, sin = ropes[layer.kind] | |
| values = layer( | |
| values, | |
| cos=cos, | |
| sin=sin, | |
| attn_bias=biases[layer.kind], | |
| value_scale=value_scale, | |
| ) | |
| if index in self.config.tap_layers: | |
| taps[index] = values | |
| low, middle, high = (taps[index] for index in self.config.tap_layers) | |
| return EncoderTaps(low=low, middle=middle, high=high) | |