qwen3vl-8b

Grill run history

llama-swap.yaml entry

  "qwen3vl-8b":
    # Qwen3-VL-8B-Instruct (Q4_K_M, 4.7 GB + 1.1 GB mmproj). Added 2026-08-23 as
    # the smallest capable VISION model for co-residency — the 4060 Ti swap pool
    # (mellum, gemma12, bonsai, glm-ocr) has no vision model of its own; this
    # fills that gap without needing both cards (unlike the 30B AWQ qwen3-vl
    # entries that are TP=2 and evict everything, including the FreeToken driver).
    #
    # NON-THINKING (Instruct, not Thinking): no reasoning_content, no
    # --reasoning-parser needed. Qwen3VLForConditionalGeneration, 36 layers,
    # 8 KV heads, 4096 hidden. Native 262k ctx but capped at 131072 here.
    #
    # KV: q4_0 for both K and V — at 131072 ctx that is ~4.8 GB, leaving ~5 GB
    # headroom for the lazily-allocated ViT buffer on first image (the same
    # buffer that hard-aborts larger vision models; see [[qwen38-27b-install]]).
    # qwen3vl accepts q4_0 V cache (standard GQA attention, not MLA).
    #
    # Pinned to the 4060 Ti (CUDA_VISIBLE_DEVICES=1, PCI_BUS_ID ordering — device
    # 1 is the 4060 Ti; see [[cuda-device-order-pci-bus-id]]). Shares the card
    # with the rest of the swap pool; only one fits at a time.
    #
    # NOT YET GRILLED — added for co-resident vision, not as a coding model.
    # Mainline ${llama_bin} (build 10588 supports qwen3vl natively).
    cmd: |
      /usr/bin/env
      CUDA_DEVICE_ORDER=PCI_BUS_ID
      CUDA_VISIBLE_DEVICES=1
      ${llama_bin}
      -m /mnt/models/gguf/Qwen3VL-8B-Instruct-Q4_K_M.gguf
      --mmproj /mnt/models/gguf/mmproj-Qwen3VL-8B-Instruct-F16.gguf
      --alias qwen3vl-8b
      --jinja
      -ngl 99 -c 131072 -fa on
      -b 2048 -ub 512
      --cache-type-k q4_0 --cache-type-v q4_0
      --temp 0.7 --top-p 0.95 --top-k 20
      --host 127.0.0.1 --port 9164 --parallel 1
    proxy: http://127.0.0.1:9164