| """Refinement-count sweep on a controller checkpoint, at several execution lengths.
|
|
|
| Exists because ``run_ablations.py:67`` hardcodes ``data/runs/controller/controller.pt``
|
| -- the *terminal-only* controller that ``progress.md`` records as superseded. Every
|
| K-sweep row in ``data/runs/eval/results.jsonl`` is therefore measured on a checkpoint
|
| with a known-broken objective, and the arrival+hold controller that replaced it
|
| (``data/runs/ah_hold0.5``, 94% at rh=1) has never been swept at all.
|
|
|
| The question this answers: does refinement count degrade success for a controller
|
| that already does joint whole-plan refinement over a full ``rollout_plan``? If it
|
| does, "causal one-step commitment" cannot be the cause of the recursive planner's
|
| T-inversion, because this architecture has no causal commitment to begin with.
|
|
|
| Each row goes through the unmodified ``eval_controller.py``, so it shares the
|
| harness, the wrappers and -- at a fixed seed -- the held-out start/goal pairs with
|
| every historical row. Results land in their own directory; the historical files are
|
| not appended to.
|
| """
|
|
|
| import argparse
|
| import subprocess
|
| import sys
|
| from pathlib import Path
|
|
|
| PY = sys.executable
|
| ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
| def parse_args():
|
| p = argparse.ArgumentParser()
|
| p.add_argument(
|
| '--controller', default='data/runs/ah_hold0.5/controller.pt'
|
| )
|
| p.add_argument('--refinements', type=int, nargs='+',
|
| default=[0, 1, 2, 3, 5, 8])
|
| p.add_argument('--receding-horizon', type=int, nargs='+', default=[1, 5])
|
| p.add_argument('--seeds', type=int, nargs='+', default=[42, 43, 44])
|
| p.add_argument('--goal-offset', type=int, nargs='+', default=[25])
|
| p.add_argument('--num-eval', type=int, default=50)
|
| p.add_argument('--eval-budget', type=int, default=50)
|
| p.add_argument('--out', default='data/runs/eval_refine_sweep')
|
| return p.parse_args()
|
|
|
|
|
| def main():
|
| args = parse_args()
|
| total = (
|
| len(args.refinements) * len(args.receding_horizon)
|
| * len(args.seeds) * len(args.goal_offset)
|
| )
|
| done = 0
|
|
|
| for q in args.goal_offset:
|
| for rh in args.receding_horizon:
|
| for seed in args.seeds:
|
| for k in args.refinements:
|
| done += 1
|
| cmd = [
|
| PY, 'scripts/eval_controller.py',
|
| '--controller', args.controller,
|
| '--refinements', str(k),
|
| '--receding-horizon', str(rh),
|
| '--goal-offset', str(q),
|
| '--seed', str(seed),
|
| '--num-eval', str(args.num_eval),
|
| '--eval-budget', str(args.eval_budget),
|
| '--out', args.out,
|
| ]
|
| print(
|
| f'\n[{done}/{total}] q={q} rh={rh} seed={seed} K={k}',
|
| flush=True,
|
| )
|
|
|
| code = subprocess.run(cmd, cwd=ROOT).returncode
|
| if code != 0:
|
| print(f'!! exited {code}, continuing', flush=True)
|
|
|
| print(f'\nappended to {Path(args.out) / "results.jsonl"}')
|
|
|
|
|
| if __name__ == '__main__':
|
| main()
|
|
|
