qwen3-vision

Grill run history

llama-swap.yaml entry

  "qwen3-vision":
    # WINDOW 176128 restored 2026-08-18 when vLLM was ROLLED BACK to 0.26.0.
    # Under 0.27.1 this entry missed by ~10 MB ("176128 needs 4.03 GiB KV,
    # available 4.02") and had to run at 172032. Raising utilization was NOT an
    # option: already 0.97, the highest here, and this is a VISION model whose
    # image buffers are allocated dynamically. If vLLM is ever upgraded again,
    # expect ~2-3% more memory use and this entry to need 172032 again --
    # see auto-memory vllm-0271-upgrade.
    # Instruct sibling of the above — reads images BETTER (10/12 vs 8/12 ground
    # truth) but is the LEAST repeatable model measured here (spread 6;
    # klayout-apihelp swung 7/8 -> 5/8 between repeats) and manages only 2/4 on
    # the closed loop. Prefer it for plain screenshot reading, not for KLayout.
    # TOOL PARSER hermes (JSON-in-XML), NOT qwen3_coder. No <think>, so no
    # reasoning parser.
    cmd: |
      /usr/bin/env
      HF_HOME=/home/giovanni/.cache/huggingface
      CUDA_DEVICE_ORDER=PCI_BUS_ID
      PATH=/home/giovanni/vllm-env/bin:/home/giovanni/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
      ${vllm_bin} serve QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ
      --served-model-name qwen3-vision
      --tensor-parallel-size 2
      --host 127.0.0.1 --port 9132
      --max-model-len 176128
      --max-num-seqs 16
      --gpu-memory-utilization 0.97
      --kv-cache-dtype fp8
      --override-generation-config '{"temperature":0.7,"repetition_penalty":1.0}'
      --enable-auto-tool-choice
      --tool-call-parser hermes
    proxy: http://127.0.0.1:9132