tiny-random/d3-turbo

This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from vllm-sr/d3-turbo.

Structural comparison with vllm-sr/d3-turbo:

Feature	Source	Tiny
Task / public interface	D3 System One decision	same system_one() interface
Input modalities	text, image, video	same
Vocabulary rows / embedding dtype	248,320 / BF16	same
Text layers and cadence	64; 3 linear + 1 full	8; two cycles
Text width / attention head dimension	5,120 / 256	32 / 32
Linear-attention K/V heads and dimensions	16 / 48; 128 / 128	1 / 1; 32 / 32
Text MLP width	17,408	64
Vision depth / width / output width	27 / 1,152 / 5,120	2 / 32 / 32
Decision readout	255-way FP32	same
Stored precision	BF16 backbone + FP32 readout	same

This is a D3 decision checkpoint, not a text-generation checkpoint. Its source runtime supplies prefix-bidirectional full attention on top of the causal Gated DeltaNet layers, and the tiny package preserves its raw runtime, decision configuration, tokenizer, image processor, and video processor.

File path	Size
model.safetensors	16.4MB
readout.safetensors	0.0MB

Example usage:

• Transformers (✅ verified)

import numpy as np
import torch
from PIL import Image
from transformers import AutoModel

model_id = "tiny-random/d3-turbo"
# The pinned D3 runtime calls torch.cuda.set_device() during construction,
# which requires an indexed CUDA device rather than the shorthand "cuda".
device = "cuda:0" if torch.cuda.is_available() else "cpu"
model = AutoModel.from_pretrained(
    model_id, trust_remote_code=True, device=device, verify="none",
).eval()
image = Image.new("RGB", (64, 64), "red")
video = np.stack([
    np.asarray(Image.new("RGB", (64, 64), color))
    for color in ("red", "green", "blue", "yellow")
])
result = model.system_one(
    state="A small red square changes color over time.",
    questions={
        "route": {
            "type": "choice",
            "instructions": "Choose the better description.",
            "criteria": {"still": "a still red image", "motion": "a changing video"},
        }
    },
    images=[image],
    videos=[video],
)
print(result)

Codes to create this repo (✅ verified):

Click to expand

import json
from pathlib import Path
from shutil import copyfile

import torch
from huggingface_hub import hf_hub_download, list_repo_files
from safetensors.torch import save_file
from transformers import AutoConfig, Qwen3_5Model, set_seed

source_model_id = "vllm-sr/d3-turbo"
save_folder = "/tmp/tiny-random/d3-turbo"
source_revision = "a0f3e9e59ebd1202fe1d4de1c4c0f9a986758cbb"
set_seed(42)
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(
    source_model_id, "config.json", revision=source_revision,
)).read_text(encoding="utf-8"))
text = raw["text_config"]
# D3 uses the stock Qwen3.5 backbone directly: retain every one of its
# 64 layers and its three linear-attention then one full-attention cadence.
text.update(
    hidden_size=32,
    head_dim=32,
    num_attention_heads=1,
    num_key_value_heads=1,
    intermediate_size=64,
    linear_key_head_dim=32,
    linear_value_head_dim=32,
    linear_num_key_heads=1,
    linear_num_value_heads=1,
    num_hidden_layers=8,
    layer_types=["linear_attention", "linear_attention", "linear_attention", "full_attention"] * 2,
)
text["rope_parameters"]["mrope_section"] = [1, 1, 2]
raw["vision_config"].update(
    hidden_size=32,
    intermediate_size=64,
    num_heads=1,
    out_hidden_size=32,
    depth=2,
)
(folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
config = AutoConfig.from_pretrained(folder)
model = Qwen3_5Model(config).to(torch.bfloat16)
with torch.no_grad():
    for name, parameter in sorted(model.named_parameters()):
        if not parameter.is_floating_point():
            continue
        # Qwen3.5 RMSNorm implements (1 + weight); its source checkpoint
        # stores these parameters in BF16 around the zero-centered form.
        if name.startswith("language_model") and (
            name.endswith("input_layernorm.weight")
            or name.endswith("post_attention_layernorm.weight")
            or name.endswith(".linear_attn.norm.weight")
            or name.endswith("language_model.norm.weight")
            or name.endswith(".self_attn.q_norm.weight")
            or name.endswith(".self_attn.k_norm.weight")
        ):
            parameter.zero_()
        elif name.endswith((".linear_attn.A_log", ".linear_attn.dt_bias")):
            # Keep the native DeltaNet initialization: it is not an
            # ordinary projection weight and controls recurrent decay.
            continue
        elif ".norm" in name and name.endswith("weight"):
            parameter.fill_(1)
        else:
            torch.nn.init.normal_(parameter, mean=0.0, std=0.02)

# The D3 package stores its decision head separately from the backbone.
# Build a real module tree solely for the README's faithful model printout,
# then serialize the exact source layout below.
package = torch.nn.Module()
package.backbone = model
package.readout = torch.nn.Linear(text["hidden_size"], 255, bias=False, dtype=torch.float32)
torch.nn.init.normal_(package.readout.weight, mean=0.0, std=0.02)
model.save_pretrained(folder, safe_serialization=True)
save_file({"weight": package.readout.weight.detach().cpu().contiguous()}, folder / "readout.safetensors")

# The pinned D3 runtime, tokenizer, chat template, and image/video
# processor are part of the executable source interface.  Copy them as raw
# bytes after saving so none of the source assets is reserialized.
assets = {
    "chat_template.jinja", "d3_compose.py", "d3_engine.py", "d3_fast.py",
    "d3_format.py", "d3_kernels.py", "d3_runtime.py", "d3_server.py",
    "decision_config.json", "merges.txt", "modeling_d3.py", "pipeline_d3.py",
    "preprocessor_config.json", "requirements.txt", "tokenizer.json",
    "tokenizer_config.json", "video_preprocessor_config.json", "vocab.json",
}
for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
    copyfile(
        hf_hub_download(source_model_id, filename, revision=source_revision),
        folder / filename,
    )

Printing the model:

Click to expand

Module(
  (backbone): Qwen3_5Model(
    (visual): Qwen3_5VisionModel(
      (patch_embed): Qwen3_5VisionPatchEmbed(
        (proj): Conv3d(3, 32, kernel_size=(2, 16, 16), stride=(2, 16, 16))
      )
      (pos_embed): Embedding(2304, 32)
      (rotary_pos_emb): Qwen3_5VisionRotaryEmbedding()
      (blocks): ModuleList(
        (0-1): 2 x Qwen3_5VisionBlock(
          (norm1): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
          (norm2): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
          (attn): Qwen3_5VisionAttention(
            (qkv): Linear(in_features=32, out_features=96, bias=True)
            (proj): Linear(in_features=32, out_features=32, bias=True)
          )
          (mlp): Qwen3_5VisionMLP(
            (linear_fc1): Linear(in_features=32, out_features=64, bias=True)
            (linear_fc2): Linear(in_features=64, out_features=32, bias=True)
            (act_fn): GELUTanh()
          )
        )
      )
      (merger): Qwen3_5VisionPatchMerger(
        (norm): LayerNorm((32,), eps=1e-06, elementwise_affine=True, bias=True)
        (linear_fc1): Linear(in_features=128, out_features=128, bias=True)
        (act_fn): GELU(approximate='none')
        (linear_fc2): Linear(in_features=128, out_features=32, bias=True)
      )
    )
    (language_model): Qwen3_5TextModel(
      (embed_tokens): Embedding(248320, 32)
      (layers): ModuleList(
        (0-2): 3 x Qwen3_5DecoderLayer(
          (linear_attn): Qwen3_5GatedDeltaNet(
            (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
            (norm): Qwen3_5RMSNormGated()
            (out_proj): Linear(in_features=32, out_features=32, bias=False)
            (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
            (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
            (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
            (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
          )
          (mlp): Qwen3_5MLP(
            (gate_proj): Linear(in_features=32, out_features=64, bias=False)
            (up_proj): Linear(in_features=32, out_features=64, bias=False)
            (down_proj): Linear(in_features=64, out_features=32, bias=False)
            (act_fn): SiLUActivation()
          )
          (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
          (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
        )
        (3): Qwen3_5DecoderLayer(
          (self_attn): Qwen3_5Attention(
            (q_proj): Linear(in_features=32, out_features=64, bias=False)
            (k_proj): Linear(in_features=32, out_features=32, bias=False)
            (v_proj): Linear(in_features=32, out_features=32, bias=False)
            (o_proj): Linear(in_features=32, out_features=32, bias=False)
            (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
            (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
          )
          (mlp): Qwen3_5MLP(
            (gate_proj): Linear(in_features=32, out_features=64, bias=False)
            (up_proj): Linear(in_features=32, out_features=64, bias=False)
            (down_proj): Linear(in_features=64, out_features=32, bias=False)
            (act_fn): SiLUActivation()
          )
          (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
          (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
        )
        (4-6): 3 x Qwen3_5DecoderLayer(
          (linear_attn): Qwen3_5GatedDeltaNet(
            (conv1d): Conv1d(96, 96, kernel_size=(4,), stride=(1,), padding=(3,), groups=96, bias=False)
            (norm): Qwen3_5RMSNormGated()
            (out_proj): Linear(in_features=32, out_features=32, bias=False)
            (in_proj_qkv): Linear(in_features=32, out_features=96, bias=False)
            (in_proj_z): Linear(in_features=32, out_features=32, bias=False)
            (in_proj_b): Linear(in_features=32, out_features=1, bias=False)
            (in_proj_a): Linear(in_features=32, out_features=1, bias=False)
          )
          (mlp): Qwen3_5MLP(
            (gate_proj): Linear(in_features=32, out_features=64, bias=False)
            (up_proj): Linear(in_features=32, out_features=64, bias=False)
            (down_proj): Linear(in_features=64, out_features=32, bias=False)
            (act_fn): SiLUActivation()
          )
          (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
          (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
        )
        (7): Qwen3_5DecoderLayer(
          (self_attn): Qwen3_5Attention(
            (q_proj): Linear(in_features=32, out_features=64, bias=False)
            (k_proj): Linear(in_features=32, out_features=32, bias=False)
            (v_proj): Linear(in_features=32, out_features=32, bias=False)
            (o_proj): Linear(in_features=32, out_features=32, bias=False)
            (q_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
            (k_norm): Qwen3_5RMSNorm((32,), eps=1e-06)
          )
          (mlp): Qwen3_5MLP(
            (gate_proj): Linear(in_features=32, out_features=64, bias=False)
            (up_proj): Linear(in_features=32, out_features=64, bias=False)
            (down_proj): Linear(in_features=64, out_features=32, bias=False)
            (act_fn): SiLUActivation()
          )
          (input_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
          (post_attention_layernorm): Qwen3_5RMSNorm((32,), eps=1e-06)
        )
      )
      (norm): Qwen3_5RMSNorm((32,), eps=1e-06)
      (rotary_emb): Qwen3_5TextRotaryEmbedding()
    )
  )
  (readout): Linear(in_features=32, out_features=255, bias=False)
)

Test environment:

• Pillow: 13.0.0
• huggingface_hub: 2.3.0
• numpy: 2.3.5
• safetensors: 0.8.0
• tokenizers: 0.23.3
• torch: 2.13.0
• transformers: 5.19.0

Change log:

• 2026-10-23: Initial version.

This repo is generated by GPT-6.1-Sol