"""Prompt tokenization for the T5Gemma2 text encoder, exactly as in training. - The tokenizer's special-token frame (````) wraps every prompt. - Text the image should show is tokenized one character at a time, so its spelling becomes token identity: in a plain prompt every straight (``"..."``) or curly (``“...”``) double-quoted span on one line, quote marks included; in a JSON layout prompt the string value of every ``"text"`` key and every escaped-quote (``\\"...\\"``) span inside other string values. - Everything else is tokenized normally. - A prompt over ``MAX_PROMPT_TOKENS`` tokens (frame included) raises; prompts are never truncated. """ from __future__ import annotations import re from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from transformers import AutoTokenizer, PreTrainedTokenizerBase from .layout import is_compiled_layout __all__ = [ "MAX_PROMPT_TOKENS", "PromptTokenizer", "PromptTooLongError", "TokenizedPrompt", "quoted_segments", "structured_segments", ] # The longest prompt the model was trained on, in tokens (special tokens included). MAX_PROMPT_TOKENS = 6144 _QUOTED_SPAN = re.compile(r"\"[^\"\n]*\"|“[^”\n]*”") # A JSON string literal (quote marks included) and one escape sequence in it. _JSON_STRING = re.compile(r"\"(?:[^\"\\]|\\.)*\"") _JSON_ESCAPE = re.compile(r"\\.") _JSON_KEY_COLON = re.compile(r"\s*:") _JSON_ESCAPED_QUOTE = '\\"' # The JSON key whose string value is visible text (a layout item's ``text``). _VISIBLE_TEXT_KEY = '"text"' # Probe text used to find the special tokens the tokenizer adds around text. _FRAME_PROBE = "a" type Segment = tuple[int, int, bool] type Offset = tuple[int, int] class PromptTooLongError(ValueError): """A prompt has more tokens than the model was trained on.""" @dataclass(frozen=True) class TokenizedPrompt: """Token ids of one prompt and each token's character span in it. Attributes: ids: Token ids, special tokens included. offsets: Per token, its ``(start, end)`` character span in the prompt; ``(0, 0)`` for special tokens. """ ids: tuple[int, ...] offsets: tuple[Offset, ...] def _segments_from_spans(text: str, spans: Sequence[Offset]) -> list[Segment]: """Cover ``text`` with ``(start, end, in_span)`` segments for ordered, non-overlapping ``spans``.""" segments: list[Segment] = [] cursor = 0 for start, end in spans: if start > cursor: segments.append((cursor, start, False)) segments.append((start, end, True)) cursor = end if cursor < len(text): segments.append((cursor, len(text), False)) return segments def quoted_segments(text: str) -> list[Segment]: """Split a plain prompt into ``(start, end, is_quoted)`` segments covering it. Quoted segments are balanced straight or curly double quotes on one line, quote marks included. Single quotes are not quotes (apostrophes). """ return _segments_from_spans( text, [match.span() for match in _QUOTED_SPAN.finditer(text)] ) def _escaped_quote_spans(literal: str, *, offset: int) -> list[Offset]: """Spans of balanced ``\\"...\\"`` pairs inside one JSON string literal, shifted by ``offset``; an unpaired last escaped quote is ignored.""" quotes = [ match.start() for match in _JSON_ESCAPE.finditer(literal) if match.group(0) == _JSON_ESCAPED_QUOTE ] return [ (offset + quotes[pos], offset + quotes[pos + 1] + len(_JSON_ESCAPED_QUOTE)) for pos in range(0, len(quotes) - 1, 2) ] def structured_segments(text: str) -> list[Segment]: """Split a JSON layout prompt into ``(start, end, is_visible_text)`` segments. Visible text is the string value of every ``"text"`` key (quote marks included) and each balanced ``\\"...\\"`` span inside any other string value. String literals are lexed whole. Raises: ValueError: If ``text`` has a double quote outside a string literal. """ spans: list[Offset] = [] cursor = 0 after_text_key = False for match in _JSON_STRING.finditer(text): start, end = match.span() between = text[cursor:start] if '"' in between: raise ValueError( f"Layout prompt has an unterminated string at {cursor}: {text[:120]!r}" ) if _JSON_KEY_COLON.match(text, end) is not None: after_text_key = match.group(0) == _VISIBLE_TEXT_KEY elif after_text_key and between.strip() == ":": spans.append((start, end)) after_text_key = False else: spans.extend(_escaped_quote_spans(match.group(0), offset=start)) after_text_key = False cursor = end if '"' in text[cursor:]: raise ValueError( f"Layout prompt has an unterminated string at {cursor}: {text[:120]!r}" ) return _segments_from_spans(text, spans) def _int_rows(raw: object, *, name: str) -> list[list[int]]: """Return a tokenizer output field as integer rows. Raises: TypeError: If the field is not a list of integer lists. """ match raw: case list() if all( isinstance(row, list) and all(isinstance(v, int) for v in row) for row in raw ): return [[int(v) for v in row] for row in raw] case _: raise TypeError(f"Tokenizer {name} must be a list of integer lists.") def _offset_rows(raw: object) -> list[list[Offset]]: """Return tokenizer offset mappings as ``(start, end)`` rows. Raises: TypeError: If the field is not a list of integer-pair lists. """ match raw: case list(): rows: list[list[Offset]] = [] for row in raw: match row: case list(): pairs: list[Offset] = [] for pair in row: match pair: case (int() as start, int() as end): pairs.append((start, end)) case [int() as start, int() as end]: pairs.append((start, end)) case _: raise TypeError( "Tokenizer offsets must be integer pairs." ) rows.append(pairs) case _: raise TypeError("Tokenizer offsets must be rows of pairs.") return rows case _: raise TypeError("Tokenizer offset_mapping must be a list of rows.") class PromptTokenizer: """The T5Gemma2 tokenizer with the training-time prompt rules. Args: tokenizer: The fast T5Gemma2 tokenizer. max_tokens: The longest allowed prompt in tokens. Raises: RuntimeError: If the special-token frame cannot be located. """ def __init__( self, tokenizer: PreTrainedTokenizerBase, *, max_tokens: int = MAX_PROMPT_TOKENS ) -> None: if max_tokens <= 0: raise ValueError(f"max_tokens must be positive, got {max_tokens}.") self.tokenizer = tokenizer self.max_tokens = max_tokens self.prefix_ids, self.suffix_ids = self._special_token_frame() @classmethod def from_directory(cls, directory: str | Path) -> PromptTokenizer: """Load the tokenizer files of a bundled ``text_encoder/`` folder. Raises: FileNotFoundError: If ``tokenizer.json`` is missing. """ path = Path(directory) if not (path / "tokenizer.json").is_file(): raise FileNotFoundError(f"No tokenizer.json in {path}.") # The folder's config.json is ours (no transformers version), which # transformers would otherwise take for an old Mistral checkpoint. loaded = AutoTokenizer.from_pretrained( str(path), local_files_only=True, fix_mistral_regex=False ) if not isinstance(loaded, PreTrainedTokenizerBase) or not loaded.is_fast: raise TypeError(f"{path} does not hold a fast tokenizer.") return cls(loaded) def _encode(self, texts: Sequence[str], *, special: bool) -> list[list[int]]: """Return the plain token ids of each text.""" encoded = self.tokenizer( list(texts), padding=False, truncation=False, add_special_tokens=special, ) return _int_rows(encoded["input_ids"], name="input_ids") def _special_token_frame(self) -> tuple[tuple[int, ...], tuple[int, ...]]: """Return the special-token ids the tokenizer adds before and after text. Raises: RuntimeError: If the plain ids are not a run inside the framed ids. """ plain = self._encode([_FRAME_PROBE], special=False)[0] framed = self._encode([_FRAME_PROBE], special=True)[0] width = len(plain) for start in range(len(framed) - width + 1): if framed[start : start + width] == plain: return tuple(framed[:start]), tuple(framed[start + width :]) raise RuntimeError("Could not locate plain tokens inside the special frame.") def tokenize(self, text: str) -> TokenizedPrompt: """Tokenize one prompt (plain or a compiled JSON layout prompt). Raises: PromptTooLongError: If the prompt exceeds ``max_tokens``. ValueError: If a JSON layout prompt has an unterminated string. """ segments = ( structured_segments(text) if is_compiled_layout(text) else quoted_segments(text) ) pieces: list[str] = [] starts: list[int] = [] for start, end, by_character in segments: if by_character: pieces.extend(text[start:end]) starts.extend(range(start, end)) else: pieces.append(text[start:end]) starts.append(start) ids: list[int] = list(self.prefix_ids) offsets: list[Offset] = [(0, 0)] * len(self.prefix_ids) if pieces: encoded = self.tokenizer( pieces, padding=False, truncation=False, add_special_tokens=False, return_offsets_mapping=True, ) id_rows = _int_rows(encoded["input_ids"], name="input_ids") offset_rows = _offset_rows(encoded["offset_mapping"]) for base, row_ids, row_offsets in zip( starts, id_rows, offset_rows, strict=True ): ids.extend(row_ids) offsets.extend((base + lo, base + hi) for lo, hi in row_offsets) ids.extend(self.suffix_ids) offsets.extend([(0, 0)] * len(self.suffix_ids)) if len(ids) > self.max_tokens: raise PromptTooLongError( f"The prompt has {len(ids)} tokens; the model takes at most " f"{self.max_tokens}. Shorten it: {text[:120]!r}" ) return TokenizedPrompt(ids=tuple(ids), offsets=tuple(offsets))