{
  "data_dir": "dataset/actor/",
  "only_actor": "all",
  "base_model": "Qwen/Qwen2.5-7B-Instruct",
  "output_dir": "out/",
  "gamma": 0.99,
  "tau": 0.005,
  "beta_dpo": 1,
  "beta_q": 1.0,
  "beta_sft": 0.5,
  "num_candidates": 3,
  "k_next": 3,
  "actor_gen_max_new": 30,
  "next_gen_max_new": 30,
  "gen_temperature": 1.5,
  "beta_format_penalty": 1.0,
  "format_penalty_value": -4.0,
  "total_steps": 200000,
  "sft_warmup_steps": 2000,
  "q_pretrain_steps": 2000,
  "valid_every_steps": 2000,
  "valid_n_batches": 20,
  "eval_every_steps": 245645,
  "save_every_steps": 2000,
  "batch_size": 8,
  "grad_accum_steps": 1,
  "lr": 3e-05,
  "lr_q": 0.0003,
  "weight_decay": 0.01,
  "warmup_ratio": 0.001,
  "max_grad_norm": 1.0,
  "max_length": 2048,
  "use_chat_template": false,
  "valid_ratio": 0.0,
  "log_every": 5,
  "color_log": true,
  "seed": 42,
  "lora_r": 16,
  "lora_alpha": 32,
  "lora_r_q": 16,
  "lora_alpha_q": 32,
  "lora_dropout": 0.05,
  "target_modules": [
    "q_proj",
    "k_proj",
    "v_proj",
    "o_proj",
    "gate_proj",
    "up_proj",
    "down_proj"
  ],
  "fp16": false,
  "bf16": true,
  "gradient_checkpointing": false,
  "layout": "new_env",
  "eval_verbose": false,
  "gpu_id": 0,
  "test_prompt_dir": "prompts/test/",
  "wandb": true,
  "wandb_project": "overcooked-qdpo",
  "wandb_run_name": "3phase-sft-q-dpo"
}