fable-711-gptq

Grill run history

llama-swap.yaml entry

  "fable-711-gptq":
    # The most REPEATABLE model ever measured on this box — spread 0 across all
    # eleven suites, 28/29 coding twice — but DENSE, so ~28 tok/s, a 3x penalty
    # against glm-flash for a 2-task edge. Its closed-loop 6/8 was refuted (0/4
    # from memory, 1/4 even with the API reference, with three 562s runaways).
    # Pick it when you want a careful second opinion, not as a driver.
    #
    # STORAGE TIER: back on /mnt/ssk500 as of 2026-08-16 (18GB freed on NVMe).
    # HF cache dir is a SYMLINK to the ssk500 copy again -- the 2026-08-14 NVMe
    # promotion is reversed. Reason: [[gemma-4-26b-awq-vllm]] beats this model
    # on coding (44/46 vs 22/23), speed (93 vs 28 tok/s), vision (10/12 vs
    # 4/6), and reliability (0 runaways vs 3 documented token-cap burns); this
    # entry's only remaining edge is KLayout+ref (87.5% vs 75%). At 28 tok/s
    # it is a deliberate, occasional pick, not a driver -- the ssk500 slow-load
    # penalty (~+60-100s cold) is cheap against that usage pattern.
    cmd: |
      /usr/bin/env
      HF_HOME=/home/giovanni/.cache/huggingface
      CUDA_DEVICE_ORDER=PCI_BUS_ID
      PATH=/home/giovanni/vllm-env/bin:/home/giovanni/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
      ${vllm_bin} serve cjxzdzh/Qwen3.6-27B-Fable-Fusion-711-Uncensored-Heretic-NM-DAU-MTP-GPTQ-INT4
      --served-model-name fable-711-gptq
      --tensor-parallel-size 2
      --host 127.0.0.1 --port 9134
      --max-model-len 155648
      --max-num-seqs 8
      --gpu-memory-utilization 0.90
      --kv-cache-dtype fp8
      --override-generation-config '{"temperature":0.6}'
      --enable-auto-tool-choice
      --tool-call-parser qwen3_coder
      --reasoning-parser qwen3
    proxy: http://127.0.0.1:9134