tiny-random/minimax-m3.1-flash

This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from MiniMaxAI/MiniMax-M3.1-Flash.

Structural comparison with MiniMaxAI/MiniMax-M3.1-Flash:

Feature	Source	Tiny
Text layers / dense prefix	60 / 3	8 / 3
Sparse attention and indexer	every layer	every layer
Routed experts / top-k / shared	128 / 4 / 1	128 / 4 / 1
Hidden / attention head / index head	6144 / 128 / 128	32 / 32 / 32
Vision layers / width	32 / 1280	2 / 64
Draft branch	5-stage DSpark	5-stage DSpark
Dense / routed storage	MXFP8 / NVFP4	MXFP8 / NVFP4

This is a direct source-schema generator. Native Transformers can instantiate the M3 VL graph, but cannot serialize the released NVFP4 routed-expert layout. The checkpoint follows the official SGLang loader's source names and quantization formats instead. It preserves the original vocabulary rows and untied BF16 token matrices, so those two tensors alone require 25,608,192 bytes; the complete tiny checkpoint is intentionally larger than the 30 MB soft target.

The available H20 validation fleet does not provide the Blackwell NVFP4 execution path required by the source experts. The SGLang TP=4 command is therefore listed as unverified rather than being presented as a successful test.

File path	Size
dspark/model.safetensors	6.5MB
model.safetensors	28.3MB

Example usage:

• SGLang (unverified)

python -m sglang.launch_server \
  --model-path tiny-random/minimax-m3.1-flash \
  --tp-size 4 \
  --trust-remote-code

Codes to create this repo:

Click to expand

"""Write the source M3.1 Flash checkpoint schema without changing its formats.

Transformers provides a native M3 VL module graph, but it cannot serialize the
source's NVFP4 routed experts.  The official SGLang loader is consequently the
authoritative representation for the final checkpoint names and packing.
"""
import json
import shutil
from pathlib import Path

import torch
from huggingface_hub import hf_hub_download
from safetensors.torch import save_file

source_model_id = "MiniMaxAI/MiniMax-M3.1-Flash"
save_folder = "/tmp/tiny-random/minimax-m31-flash"
revision = "54e4e69dfc452b1e94a426c24a4fdeeaa8e3ad5e"
output = Path(save_folder)
output.mkdir(parents=True, exist_ok=True)
torch.manual_seed(42)

def fetch(filename: str) -> Path:
    return Path(hf_hub_download(source_model_id, filename, revision=revision))

# Tokenizer, processors and source configuration code are copied byte-for-byte.
for filename in (
    "added_tokens.json", "chat_template.jinja", "generation_config.json",
    "merges.txt", "preprocessor_config.json", "special_tokens_map.json",
    "tokenizer.json", "tokenizer_config.json", "vocab.json",
    "configuration_minimax_m3_vl.py", "image_processor.py",
    "processing_minimax.py", "video_processor.py",
):
    shutil.copy2(fetch(filename), output / filename)

config = json.loads(fetch("config.json").read_text(encoding="utf-8"))
text = config["text_config"]
layers = 8
hidden = 32
experts = 128
text.update({
    "hidden_size": hidden,
    "intermediate_size": 32,
    "num_hidden_layers": layers,
    "num_attention_heads": 1,
    "num_key_value_heads": 1,
    "head_dim": 32,
    "dense_intermediate_size": 64,
    "shared_intermediate_size": 32,
    "rotary_dim": 32,
    "num_local_experts": experts,
    "num_experts_per_tok": 4,
    "moe_layer_freq": [0, 0, 0] + [1] * (layers - 3),
})
sparse = text["sparse_attention_config"]
sparse.update({
    "sparse_index_dim": 32,
    "sparse_num_index_heads": 1,
    "sparse_block_size": 32,
    "sparse_topk_blocks": 1,
    "sparse_local_block": 1,
    "sparse_attention_freq": [1] * layers,
    "sparse_disable_index_value": [1] * layers,
})
vision = config["vision_config"]
vision.update({
    "hidden_size": 64,
    "intermediate_size": 128,
    "num_attention_heads": 2,
    "num_hidden_layers": 2,
    "projection_dim": hidden,
    "image_size": 28,
})
config["projector_hidden_size"] = hidden
# Keep the source mixed format: MXFP8 dense/attention and NVFP4 experts.
config["quantization_config"]["ignored_layers"] = [
    "language_model.model.embed_tokens", "language_model.lm_head",
    "vision_tower", "multi_modal_projector",
    *[
        f"language_model.model.layers.{index}.block_sparse_moe.gate"
        for index in range(3, layers)
    ],
]
(output / "config.json").write_text(
    json.dumps(config, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
)

state: dict[str, torch.Tensor] = {}

def bf16(name: str, *shape: int) -> None:
    state[name] = torch.empty(shape, dtype=torch.bfloat16).normal_(0, 0.1)

def f32(name: str, *shape: int) -> None:
    state[name] = torch.empty(shape, dtype=torch.float32).normal_(0, 0.1)

def mxfp8(name: str, out_dim: int, in_dim: int) -> None:
    assert in_dim % 32 == 0, (name, in_dim)
    # PyTorch intentionally has no CPU normal kernel for float8.  Sample in
    # BF16 and cast; the stored tensor still has the source E4M3 dtype.
    state[f"{name}.weight"] = torch.empty(
        out_dim, in_dim, dtype=torch.bfloat16
    ).normal_(0, 0.1).to(torch.float8_e4m3fn)
    # E8M0 exponent 127 encodes 1.0.  SGLang identifies this as UE8M0.
    state[f"{name}.weight_scale_inv"] = torch.full(
        (out_dim, in_dim // 32), 127, dtype=torch.uint8
    )

def nvfp4(name: str, out_dim: int, in_dim: int) -> None:
    """Source NVFP4 layout: two values/byte, E4M3 per 16 columns."""
    assert in_dim % 16 == 0, (name, in_dim)
    state[f"{name}.weight_packed"] = torch.randint(
        0, 256, (out_dim, in_dim // 2), dtype=torch.uint8
    )
    state[f"{name}.weight_scale"] = torch.ones(
        (out_dim, in_dim // 16), dtype=torch.float8_e4m3fn
    )
    state[f"{name}.input_global_scale"] = torch.ones((), dtype=torch.float32)
    state[f"{name}.weight_global_scale"] = torch.ones((), dtype=torch.float32)

# The source uses BF16 token matrices.  They are intentionally untied.
bf16("language_model.model.embed_tokens.weight", text["vocab_size"], hidden)
bf16("language_model.lm_head.weight", text["vocab_size"], hidden)
bf16("language_model.model.norm.weight", hidden)

def attention(prefix: str) -> None:
    for projection, out_dim in (
        ("q_proj", 32), ("k_proj", 32), ("v_proj", 32), ("o_proj", hidden),
        ("index_q_proj", 32), ("index_k_proj", 32),
    ):
        mxfp8(f"{prefix}.self_attn.{projection}", out_dim, hidden)
    for norm in ("q_norm", "k_norm", "index_q_norm", "index_k_norm"):
        bf16(f"{prefix}.self_attn.{norm}.weight", 32)

for layer in range(layers):
    prefix = f"language_model.model.layers.{layer}"
    attention(prefix)
    bf16(f"{prefix}.input_layernorm.weight", hidden)
    bf16(f"{prefix}.post_attention_layernorm.weight", hidden)
    if layer < 3:
        for projection, out_dim, in_dim in (
            ("gate_proj", 64, hidden), ("up_proj", 64, hidden),
            ("down_proj", hidden, 64),
        ):
            mxfp8(f"{prefix}.mlp.{projection}", out_dim, in_dim)
        continue

    moe = f"{prefix}.block_sparse_moe"
    f32(f"{moe}.gate.weight", experts, hidden)
    f32(f"{moe}.e_score_correction_bias", experts)
    for expert in range(experts):
        expert_prefix = f"{moe}.experts.{expert}"
        nvfp4(f"{expert_prefix}.w1", 32, hidden)
        nvfp4(f"{expert_prefix}.w2", hidden, 32)
        nvfp4(f"{expert_prefix}.w3", 32, hidden)
    shared = f"{moe}.shared_experts"
    for projection, out_dim, in_dim in (
        ("gate_proj", 32, hidden), ("up_proj", 32, hidden),
        ("down_proj", hidden, 32),
    ):
        mxfp8(f"{shared}.{projection}", out_dim, in_dim)

# The released vision branch is BF16 and retains genuine image/video features.
bf16("vision_tower.vision_model.embeddings.patch_embedding.weight", 64, 3, 2, 14, 14)
bf16("vision_tower.vision_model.pre_layrnorm.weight", 64)
bf16("vision_tower.vision_model.pre_layrnorm.bias", 64)
for layer in range(2):
    prefix = f"vision_tower.vision_model.encoder.layers.{layer}"
    for projection in ("q_proj", "k_proj", "v_proj", "out_proj"):
        bf16(f"{prefix}.self_attn.{projection}.weight", 64, 64)
        bf16(f"{prefix}.self_attn.{projection}.bias", 64)
    for norm in ("layer_norm1", "layer_norm2"):
        bf16(f"{prefix}.{norm}.weight", 64)
        bf16(f"{prefix}.{norm}.bias", 64)
    bf16(f"{prefix}.mlp.fc1.weight", 128, 64)
    bf16(f"{prefix}.mlp.fc1.bias", 128)
    bf16(f"{prefix}.mlp.fc2.weight", 64, 128)
    bf16(f"{prefix}.mlp.fc2.bias", 64)
for projection, out_dim, in_dim in (("linear_1", hidden, 64), ("linear_2", hidden, hidden)):
    bf16(f"multi_modal_projector.{projection}.weight", out_dim, in_dim)
    bf16(f"multi_modal_projector.{projection}.bias", out_dim)

save_file(state, output / "model.safetensors")

# DSpark is a separate five-stage source checkpoint.  Keep its target-layer
# cadence and MXFP8 linear representation even though the local H20 fleet
# cannot execute this source NVFP4 + DSpark serving combination.
draft_config = json.loads(fetch("dspark/config.json").read_text(encoding="utf-8"))
draft_text = draft_config["text_config"]
draft_text.update({
    "hidden_size": hidden, "intermediate_size": 32, "num_attention_heads": 1,
    "num_key_value_heads": 1, "head_dim": 32, "dense_intermediate_size": 64,
    "shared_intermediate_size": 32, "rotary_dim": 32, "dspark_markov_rank": 16,
    "dspark_ffn_hidden_size": 64,
})
draft_config.update({"projector_hidden_size": hidden, "dspark_markov_rank": 16})
draft_dir = output / "dspark"
draft_dir.mkdir(exist_ok=True)
(draft_dir / "config.json").write_text(
    json.dumps(draft_config, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
)
draft: dict[str, torch.Tensor] = {}

def add_draft_bf16(name: str, *shape: int) -> None:
    draft[name] = torch.empty(shape, dtype=torch.bfloat16).normal_(0, 0.1)

def add_draft_mxfp8(name: str, out_dim: int, in_dim: int) -> None:
    draft[f"{name}.weight"] = torch.empty(out_dim, in_dim, dtype=torch.bfloat16).normal_(0, 0.1).to(torch.float8_e4m3fn)
    draft[f"{name}.weight_scale_inv"] = torch.full((out_dim, in_dim // 32), 127, dtype=torch.uint8)

root = "language_model.model.dspark"
add_draft_bf16(f"{root}.fc.weight", hidden, hidden * 5)
add_draft_bf16(f"{root}.hidden_norm.weight", hidden)
add_draft_bf16(f"{root}.final_norm.weight", hidden)
add_draft_bf16(f"{root}.confidence_head.proj.weight", 1, hidden)
add_draft_bf16(f"{root}.confidence_head.proj.bias", 1)
add_draft_bf16(f"{root}.markov_head.markov_w1.weight", 16, hidden)
add_draft_bf16(f"{root}.markov_head.markov_w2.weight", text["vocab_size"], 16)
for layer in range(5):
    prefix = f"{root}.layers.{layer}.decoder_layer"
    for projection, out_dim in (("q_proj", 32), ("k_proj", 32), ("v_proj", 32), ("o_proj", hidden)):
        add_draft_mxfp8(f"{prefix}.self_attn.{projection}", out_dim, hidden)
    for norm in ("q_norm", "k_norm", "input_layernorm", "post_attention_layernorm"):
        add_draft_bf16(f"{prefix}.{norm}.weight", 32 if norm in ("q_norm", "k_norm") else hidden)
    for projection, out_dim, in_dim in (("gate_proj", 64, hidden), ("up_proj", 64, hidden), ("down_proj", hidden, 64)):
        add_draft_mxfp8(f"{prefix}.mlp.{projection}", out_dim, in_dim)
save_file(draft, draft_dir / "model.safetensors")

Printing the model:

Click to expand

MiniMax M3.1 Flash source-schema checkpoint (SGLang runtime required)

Test environment:

• huggingface_hub: 1.27.0
• safetensors: 0.8.0
• tokenizers: 0.23.0rc0
• torch: 2.13.0+cu126
• transformers: 5.15.0.dev0

Change log:

• 2026-10-23: Initial version.

This repo is generated by GPT-6.1-Sol