dflow_2b / dflow /tokenization.py
data-archetype's picture
dflow_2b 0.2
6daffc2
Raw History Blame Contribute Delete
11.3 kB
"""Prompt tokenization for the T5Gemma2 text encoder, exactly as in training.
- The tokenizer's special-token frame (``<bos>``) wraps every prompt.
- Text the image should show is tokenized one character at a time, so its
spelling becomes token identity: in a plain prompt every straight
(``"..."``) or curly (``“...”``) double-quoted span on one line, quote marks
included; in a JSON layout prompt the string value of every ``"text"`` key
and every escaped-quote (``\\"...\\"``) span inside other string values.
- Everything else is tokenized normally.
- A prompt over ``MAX_PROMPT_TOKENS`` tokens (frame included) raises; prompts
are never truncated.
"""
from __future__ import annotations
import re
from collections.abc import Sequence
from dataclasses import dataclass
from pathlib import Path
from transformers import AutoTokenizer, PreTrainedTokenizerBase
from .layout import is_compiled_layout
__all__ = [
"MAX_PROMPT_TOKENS",
"PromptTokenizer",
"PromptTooLongError",
"TokenizedPrompt",
"quoted_segments",
"structured_segments",
]
# The longest prompt the model was trained on, in tokens (special tokens included).
MAX_PROMPT_TOKENS = 6144
_QUOTED_SPAN = re.compile(r"\"[^\"\n]*\"|“[^”\n]*”")
# A JSON string literal (quote marks included) and one escape sequence in it.
_JSON_STRING = re.compile(r"\"(?:[^\"\\]|\\.)*\"")
_JSON_ESCAPE = re.compile(r"\\.")
_JSON_KEY_COLON = re.compile(r"\s*:")
_JSON_ESCAPED_QUOTE = '\\"'
# The JSON key whose string value is visible text (a layout item's ``text``).
_VISIBLE_TEXT_KEY = '"text"'
# Probe text used to find the special tokens the tokenizer adds around text.
_FRAME_PROBE = "a"
type Segment = tuple[int, int, bool]
type Offset = tuple[int, int]
class PromptTooLongError(ValueError):
"""A prompt has more tokens than the model was trained on."""
@dataclass(frozen=True)
class TokenizedPrompt:
"""Token ids of one prompt and each token's character span in it.
Attributes:
ids: Token ids, special tokens included.
offsets: Per token, its ``(start, end)`` character span in the prompt;
``(0, 0)`` for special tokens.
"""
ids: tuple[int, ...]
offsets: tuple[Offset, ...]
def _segments_from_spans(text: str, spans: Sequence[Offset]) -> list[Segment]:
"""Cover ``text`` with ``(start, end, in_span)`` segments for ordered,
non-overlapping ``spans``."""
segments: list[Segment] = []
cursor = 0
for start, end in spans:
if start > cursor:
segments.append((cursor, start, False))
segments.append((start, end, True))
cursor = end
if cursor < len(text):
segments.append((cursor, len(text), False))
return segments
def quoted_segments(text: str) -> list[Segment]:
"""Split a plain prompt into ``(start, end, is_quoted)`` segments covering it.
Quoted segments are balanced straight or curly double quotes on one line,
quote marks included. Single quotes are not quotes (apostrophes).
"""
return _segments_from_spans(
text, [match.span() for match in _QUOTED_SPAN.finditer(text)]
)
def _escaped_quote_spans(literal: str, *, offset: int) -> list[Offset]:
"""Spans of balanced ``\\"...\\"`` pairs inside one JSON string literal,
shifted by ``offset``; an unpaired last escaped quote is ignored."""
quotes = [
match.start()
for match in _JSON_ESCAPE.finditer(literal)
if match.group(0) == _JSON_ESCAPED_QUOTE
]
return [
(offset + quotes[pos], offset + quotes[pos + 1] + len(_JSON_ESCAPED_QUOTE))
for pos in range(0, len(quotes) - 1, 2)
]
def structured_segments(text: str) -> list[Segment]:
"""Split a JSON layout prompt into ``(start, end, is_visible_text)`` segments.
Visible text is the string value of every ``"text"`` key (quote marks
included) and each balanced ``\\"...\\"`` span inside any other string
value. String literals are lexed whole.
Raises:
ValueError: If ``text`` has a double quote outside a string literal.
"""
spans: list[Offset] = []
cursor = 0
after_text_key = False
for match in _JSON_STRING.finditer(text):
start, end = match.span()
between = text[cursor:start]
if '"' in between:
raise ValueError(
f"Layout prompt has an unterminated string at {cursor}: {text[:120]!r}"
)
if _JSON_KEY_COLON.match(text, end) is not None:
after_text_key = match.group(0) == _VISIBLE_TEXT_KEY
elif after_text_key and between.strip() == ":":
spans.append((start, end))
after_text_key = False
else:
spans.extend(_escaped_quote_spans(match.group(0), offset=start))
after_text_key = False
cursor = end
if '"' in text[cursor:]:
raise ValueError(
f"Layout prompt has an unterminated string at {cursor}: {text[:120]!r}"
)
return _segments_from_spans(text, spans)
def _int_rows(raw: object, *, name: str) -> list[list[int]]:
"""Return a tokenizer output field as integer rows.
Raises:
TypeError: If the field is not a list of integer lists.
"""
match raw:
case list() if all(
isinstance(row, list) and all(isinstance(v, int) for v in row)
for row in raw
):
return [[int(v) for v in row] for row in raw]
case _:
raise TypeError(f"Tokenizer {name} must be a list of integer lists.")
def _offset_rows(raw: object) -> list[list[Offset]]:
"""Return tokenizer offset mappings as ``(start, end)`` rows.
Raises:
TypeError: If the field is not a list of integer-pair lists.
"""
match raw:
case list():
rows: list[list[Offset]] = []
for row in raw:
match row:
case list():
pairs: list[Offset] = []
for pair in row:
match pair:
case (int() as start, int() as end):
pairs.append((start, end))
case [int() as start, int() as end]:
pairs.append((start, end))
case _:
raise TypeError(
"Tokenizer offsets must be integer pairs."
)
rows.append(pairs)
case _:
raise TypeError("Tokenizer offsets must be rows of pairs.")
return rows
case _:
raise TypeError("Tokenizer offset_mapping must be a list of rows.")
class PromptTokenizer:
"""The T5Gemma2 tokenizer with the training-time prompt rules.
Args:
tokenizer: The fast T5Gemma2 tokenizer.
max_tokens: The longest allowed prompt in tokens.
Raises:
RuntimeError: If the special-token frame cannot be located.
"""
def __init__(
self, tokenizer: PreTrainedTokenizerBase, *, max_tokens: int = MAX_PROMPT_TOKENS
) -> None:
if max_tokens <= 0:
raise ValueError(f"max_tokens must be positive, got {max_tokens}.")
self.tokenizer = tokenizer
self.max_tokens = max_tokens
self.prefix_ids, self.suffix_ids = self._special_token_frame()
@classmethod
def from_directory(cls, directory: str | Path) -> PromptTokenizer:
"""Load the tokenizer files of a bundled ``text_encoder/`` folder.
Raises:
FileNotFoundError: If ``tokenizer.json`` is missing.
"""
path = Path(directory)
if not (path / "tokenizer.json").is_file():
raise FileNotFoundError(f"No tokenizer.json in {path}.")
# The folder's config.json is ours (no transformers version), which
# transformers would otherwise take for an old Mistral checkpoint.
loaded = AutoTokenizer.from_pretrained(
str(path), local_files_only=True, fix_mistral_regex=False
)
if not isinstance(loaded, PreTrainedTokenizerBase) or not loaded.is_fast:
raise TypeError(f"{path} does not hold a fast tokenizer.")
return cls(loaded)
def _encode(self, texts: Sequence[str], *, special: bool) -> list[list[int]]:
"""Return the plain token ids of each text."""
encoded = self.tokenizer(
list(texts),
padding=False,
truncation=False,
add_special_tokens=special,
)
return _int_rows(encoded["input_ids"], name="input_ids")
def _special_token_frame(self) -> tuple[tuple[int, ...], tuple[int, ...]]:
"""Return the special-token ids the tokenizer adds before and after text.
Raises:
RuntimeError: If the plain ids are not a run inside the framed ids.
"""
plain = self._encode([_FRAME_PROBE], special=False)[0]
framed = self._encode([_FRAME_PROBE], special=True)[0]
width = len(plain)
for start in range(len(framed) - width + 1):
if framed[start : start + width] == plain:
return tuple(framed[:start]), tuple(framed[start + width :])
raise RuntimeError("Could not locate plain tokens inside the special frame.")
def tokenize(self, text: str) -> TokenizedPrompt:
"""Tokenize one prompt (plain or a compiled JSON layout prompt).
Raises:
PromptTooLongError: If the prompt exceeds ``max_tokens``.
ValueError: If a JSON layout prompt has an unterminated string.
"""
segments = (
structured_segments(text)
if is_compiled_layout(text)
else quoted_segments(text)
)
pieces: list[str] = []
starts: list[int] = []
for start, end, by_character in segments:
if by_character:
pieces.extend(text[start:end])
starts.extend(range(start, end))
else:
pieces.append(text[start:end])
starts.append(start)
ids: list[int] = list(self.prefix_ids)
offsets: list[Offset] = [(0, 0)] * len(self.prefix_ids)
if pieces:
encoded = self.tokenizer(
pieces,
padding=False,
truncation=False,
add_special_tokens=False,
return_offsets_mapping=True,
)
id_rows = _int_rows(encoded["input_ids"], name="input_ids")
offset_rows = _offset_rows(encoded["offset_mapping"])
for base, row_ids, row_offsets in zip(
starts, id_rows, offset_rows, strict=True
):
ids.extend(row_ids)
offsets.extend((base + lo, base + hi) for lo, hi in row_offsets)
ids.extend(self.suffix_ids)
offsets.extend([(0, 0)] * len(self.suffix_ids))
if len(ids) > self.max_tokens:
raise PromptTooLongError(
f"The prompt has {len(ids)} tokens; the model takes at most "
f"{self.max_tokens}. Shorten it: {text[:120]!r}"
)
return TokenizedPrompt(ids=tuple(ids), offsets=tuple(offsets))