Download dflow/tokenization.py from data-archetype/dflow_2b: direct link, hf CLI and curl.
- Browser
- Download file 11.3 kB
-
https://huggingface.co/data-archetype/dflow_2b/resolve/main/dflow/tokenization.py
- Command line
-
hf download hf://data-archetype/dflow_2b/dflow/tokenization.py
-
curl -L -o tokenization.py https://huggingface.co/data-archetype/dflow_2b/resolve/main/dflow/tokenization.py
11.3 kB
| """Prompt tokenization for the T5Gemma2 text encoder, exactly as in training. | |
| - The tokenizer's special-token frame (``<bos>``) wraps every prompt. | |
| - Text the image should show is tokenized one character at a time, so its | |
| spelling becomes token identity: in a plain prompt every straight | |
| (``"..."``) or curly (``“...”``) double-quoted span on one line, quote marks | |
| included; in a JSON layout prompt the string value of every ``"text"`` key | |
| and every escaped-quote (``\\"...\\"``) span inside other string values. | |
| - Everything else is tokenized normally. | |
| - A prompt over ``MAX_PROMPT_TOKENS`` tokens (frame included) raises; prompts | |
| are never truncated. | |
| """ | |
| from __future__ import annotations | |
| import re | |
| from collections.abc import Sequence | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from transformers import AutoTokenizer, PreTrainedTokenizerBase | |
| from .layout import is_compiled_layout | |
| __all__ = [ | |
| "MAX_PROMPT_TOKENS", | |
| "PromptTokenizer", | |
| "PromptTooLongError", | |
| "TokenizedPrompt", | |
| "quoted_segments", | |
| "structured_segments", | |
| ] | |
| # The longest prompt the model was trained on, in tokens (special tokens included). | |
| MAX_PROMPT_TOKENS = 6144 | |
| _QUOTED_SPAN = re.compile(r"\"[^\"\n]*\"|“[^”\n]*”") | |
| # A JSON string literal (quote marks included) and one escape sequence in it. | |
| _JSON_STRING = re.compile(r"\"(?:[^\"\\]|\\.)*\"") | |
| _JSON_ESCAPE = re.compile(r"\\.") | |
| _JSON_KEY_COLON = re.compile(r"\s*:") | |
| _JSON_ESCAPED_QUOTE = '\\"' | |
| # The JSON key whose string value is visible text (a layout item's ``text``). | |
| _VISIBLE_TEXT_KEY = '"text"' | |
| # Probe text used to find the special tokens the tokenizer adds around text. | |
| _FRAME_PROBE = "a" | |
| type Segment = tuple[int, int, bool] | |
| type Offset = tuple[int, int] | |
| class PromptTooLongError(ValueError): | |
| """A prompt has more tokens than the model was trained on.""" | |
| class TokenizedPrompt: | |
| """Token ids of one prompt and each token's character span in it. | |
| Attributes: | |
| ids: Token ids, special tokens included. | |
| offsets: Per token, its ``(start, end)`` character span in the prompt; | |
| ``(0, 0)`` for special tokens. | |
| """ | |
| ids: tuple[int, ...] | |
| offsets: tuple[Offset, ...] | |
| def _segments_from_spans(text: str, spans: Sequence[Offset]) -> list[Segment]: | |
| """Cover ``text`` with ``(start, end, in_span)`` segments for ordered, | |
| non-overlapping ``spans``.""" | |
| segments: list[Segment] = [] | |
| cursor = 0 | |
| for start, end in spans: | |
| if start > cursor: | |
| segments.append((cursor, start, False)) | |
| segments.append((start, end, True)) | |
| cursor = end | |
| if cursor < len(text): | |
| segments.append((cursor, len(text), False)) | |
| return segments | |
| def quoted_segments(text: str) -> list[Segment]: | |
| """Split a plain prompt into ``(start, end, is_quoted)`` segments covering it. | |
| Quoted segments are balanced straight or curly double quotes on one line, | |
| quote marks included. Single quotes are not quotes (apostrophes). | |
| """ | |
| return _segments_from_spans( | |
| text, [match.span() for match in _QUOTED_SPAN.finditer(text)] | |
| ) | |
| def _escaped_quote_spans(literal: str, *, offset: int) -> list[Offset]: | |
| """Spans of balanced ``\\"...\\"`` pairs inside one JSON string literal, | |
| shifted by ``offset``; an unpaired last escaped quote is ignored.""" | |
| quotes = [ | |
| match.start() | |
| for match in _JSON_ESCAPE.finditer(literal) | |
| if match.group(0) == _JSON_ESCAPED_QUOTE | |
| ] | |
| return [ | |
| (offset + quotes[pos], offset + quotes[pos + 1] + len(_JSON_ESCAPED_QUOTE)) | |
| for pos in range(0, len(quotes) - 1, 2) | |
| ] | |
| def structured_segments(text: str) -> list[Segment]: | |
| """Split a JSON layout prompt into ``(start, end, is_visible_text)`` segments. | |
| Visible text is the string value of every ``"text"`` key (quote marks | |
| included) and each balanced ``\\"...\\"`` span inside any other string | |
| value. String literals are lexed whole. | |
| Raises: | |
| ValueError: If ``text`` has a double quote outside a string literal. | |
| """ | |
| spans: list[Offset] = [] | |
| cursor = 0 | |
| after_text_key = False | |
| for match in _JSON_STRING.finditer(text): | |
| start, end = match.span() | |
| between = text[cursor:start] | |
| if '"' in between: | |
| raise ValueError( | |
| f"Layout prompt has an unterminated string at {cursor}: {text[:120]!r}" | |
| ) | |
| if _JSON_KEY_COLON.match(text, end) is not None: | |
| after_text_key = match.group(0) == _VISIBLE_TEXT_KEY | |
| elif after_text_key and between.strip() == ":": | |
| spans.append((start, end)) | |
| after_text_key = False | |
| else: | |
| spans.extend(_escaped_quote_spans(match.group(0), offset=start)) | |
| after_text_key = False | |
| cursor = end | |
| if '"' in text[cursor:]: | |
| raise ValueError( | |
| f"Layout prompt has an unterminated string at {cursor}: {text[:120]!r}" | |
| ) | |
| return _segments_from_spans(text, spans) | |
| def _int_rows(raw: object, *, name: str) -> list[list[int]]: | |
| """Return a tokenizer output field as integer rows. | |
| Raises: | |
| TypeError: If the field is not a list of integer lists. | |
| """ | |
| match raw: | |
| case list() if all( | |
| isinstance(row, list) and all(isinstance(v, int) for v in row) | |
| for row in raw | |
| ): | |
| return [[int(v) for v in row] for row in raw] | |
| case _: | |
| raise TypeError(f"Tokenizer {name} must be a list of integer lists.") | |
| def _offset_rows(raw: object) -> list[list[Offset]]: | |
| """Return tokenizer offset mappings as ``(start, end)`` rows. | |
| Raises: | |
| TypeError: If the field is not a list of integer-pair lists. | |
| """ | |
| match raw: | |
| case list(): | |
| rows: list[list[Offset]] = [] | |
| for row in raw: | |
| match row: | |
| case list(): | |
| pairs: list[Offset] = [] | |
| for pair in row: | |
| match pair: | |
| case (int() as start, int() as end): | |
| pairs.append((start, end)) | |
| case [int() as start, int() as end]: | |
| pairs.append((start, end)) | |
| case _: | |
| raise TypeError( | |
| "Tokenizer offsets must be integer pairs." | |
| ) | |
| rows.append(pairs) | |
| case _: | |
| raise TypeError("Tokenizer offsets must be rows of pairs.") | |
| return rows | |
| case _: | |
| raise TypeError("Tokenizer offset_mapping must be a list of rows.") | |
| class PromptTokenizer: | |
| """The T5Gemma2 tokenizer with the training-time prompt rules. | |
| Args: | |
| tokenizer: The fast T5Gemma2 tokenizer. | |
| max_tokens: The longest allowed prompt in tokens. | |
| Raises: | |
| RuntimeError: If the special-token frame cannot be located. | |
| """ | |
| def __init__( | |
| self, tokenizer: PreTrainedTokenizerBase, *, max_tokens: int = MAX_PROMPT_TOKENS | |
| ) -> None: | |
| if max_tokens <= 0: | |
| raise ValueError(f"max_tokens must be positive, got {max_tokens}.") | |
| self.tokenizer = tokenizer | |
| self.max_tokens = max_tokens | |
| self.prefix_ids, self.suffix_ids = self._special_token_frame() | |
| def from_directory(cls, directory: str | Path) -> PromptTokenizer: | |
| """Load the tokenizer files of a bundled ``text_encoder/`` folder. | |
| Raises: | |
| FileNotFoundError: If ``tokenizer.json`` is missing. | |
| """ | |
| path = Path(directory) | |
| if not (path / "tokenizer.json").is_file(): | |
| raise FileNotFoundError(f"No tokenizer.json in {path}.") | |
| # The folder's config.json is ours (no transformers version), which | |
| # transformers would otherwise take for an old Mistral checkpoint. | |
| loaded = AutoTokenizer.from_pretrained( | |
| str(path), local_files_only=True, fix_mistral_regex=False | |
| ) | |
| if not isinstance(loaded, PreTrainedTokenizerBase) or not loaded.is_fast: | |
| raise TypeError(f"{path} does not hold a fast tokenizer.") | |
| return cls(loaded) | |
| def _encode(self, texts: Sequence[str], *, special: bool) -> list[list[int]]: | |
| """Return the plain token ids of each text.""" | |
| encoded = self.tokenizer( | |
| list(texts), | |
| padding=False, | |
| truncation=False, | |
| add_special_tokens=special, | |
| ) | |
| return _int_rows(encoded["input_ids"], name="input_ids") | |
| def _special_token_frame(self) -> tuple[tuple[int, ...], tuple[int, ...]]: | |
| """Return the special-token ids the tokenizer adds before and after text. | |
| Raises: | |
| RuntimeError: If the plain ids are not a run inside the framed ids. | |
| """ | |
| plain = self._encode([_FRAME_PROBE], special=False)[0] | |
| framed = self._encode([_FRAME_PROBE], special=True)[0] | |
| width = len(plain) | |
| for start in range(len(framed) - width + 1): | |
| if framed[start : start + width] == plain: | |
| return tuple(framed[:start]), tuple(framed[start + width :]) | |
| raise RuntimeError("Could not locate plain tokens inside the special frame.") | |
| def tokenize(self, text: str) -> TokenizedPrompt: | |
| """Tokenize one prompt (plain or a compiled JSON layout prompt). | |
| Raises: | |
| PromptTooLongError: If the prompt exceeds ``max_tokens``. | |
| ValueError: If a JSON layout prompt has an unterminated string. | |
| """ | |
| segments = ( | |
| structured_segments(text) | |
| if is_compiled_layout(text) | |
| else quoted_segments(text) | |
| ) | |
| pieces: list[str] = [] | |
| starts: list[int] = [] | |
| for start, end, by_character in segments: | |
| if by_character: | |
| pieces.extend(text[start:end]) | |
| starts.extend(range(start, end)) | |
| else: | |
| pieces.append(text[start:end]) | |
| starts.append(start) | |
| ids: list[int] = list(self.prefix_ids) | |
| offsets: list[Offset] = [(0, 0)] * len(self.prefix_ids) | |
| if pieces: | |
| encoded = self.tokenizer( | |
| pieces, | |
| padding=False, | |
| truncation=False, | |
| add_special_tokens=False, | |
| return_offsets_mapping=True, | |
| ) | |
| id_rows = _int_rows(encoded["input_ids"], name="input_ids") | |
| offset_rows = _offset_rows(encoded["offset_mapping"]) | |
| for base, row_ids, row_offsets in zip( | |
| starts, id_rows, offset_rows, strict=True | |
| ): | |
| ids.extend(row_ids) | |
| offsets.extend((base + lo, base + hi) for lo, hi in row_offsets) | |
| ids.extend(self.suffix_ids) | |
| offsets.extend([(0, 0)] * len(self.suffix_ids)) | |
| if len(ids) > self.max_tokens: | |
| raise PromptTooLongError( | |
| f"The prompt has {len(ids)} tokens; the model takes at most " | |
| f"{self.max_tokens}. Shorten it: {text[:120]!r}" | |
| ) | |
| return TokenizedPrompt(ids=tuple(ids), offsets=tuple(offsets)) | |