mellum

Grill run history

llama-swap.yaml entry

  "mellum":
    # ON-DEMAND SWAP ENTRY. Was an always-on PEER on :9161 backed by the separate
    # mellum-driver systemd unit (never started/stopped/evicted by llama-swap).
    # Moved into the swap pool 2026-08-23: the driver owned the 4060 Ti outright,
    # which left glm-ocr and bonsai with nowhere to load — glm-ocr died with
    # `cudaMalloc failed: out of memory` wanting 2176 MiB of KV against 1610 free.
    # Trade: a cold load on each swap-in instead of a permanently warm model.
    #
    # Pinned to the 4060 Ti — PCI_BUS_ID ordering, so device 1 is the 4060 Ti and
    # NOT the 5060 Ti (this is the inverse of llama.cpp's default CUDA0=4060 Ti;
    # see auto-memory cuda-device-order-pci-bus-id). Shares that card with
    # gemma12 / bonsai / glm-ocr, and only one of those fits at a time, so they
    # evict each other normally.
    #
    # The mellum-driver unit still exists but is STOPPED and should stay that
    # way — it binds the same :9161 and would collide with this entry.
    cmd: |
      /usr/bin/env
      CUDA_DEVICE_ORDER=PCI_BUS_ID
      CUDA_VISIBLE_DEVICES=1
      ${llama_bin}
      -m /mnt/models/gguf/Mellum2-12B-A2.5B-Instruct-Q8_0.gguf
      --alias mellum
      --jinja
      -ngl 99 -c 131072 -fa on
      --host 127.0.0.1 --port 9161 --parallel 1
    proxy: http://127.0.0.1:9161