[Qwen3.5-27B]
model = /home/x99/gguf/Huihui-Qwen3.5-27B-Claude-4.6-Opus-abliterated-Q4_K_M.gguf
ctx-size = 153600
n-gpu-layers = 99
tensor-split = 1,1
flash-attn = on
cache-type-k = q4_0
cache-type-v = q4_0
parallel = 3
threads = 12
batch-size = 1024
ubatch-size = 1024
temp = 0.3
min-p = 0.05
repeat-penalty = 1.1
timeout = 1200
[tiny_starcoder]
model = /home/x99/gguf/tiny_starcoder_py.Q4_K_M.gguf
ctx-size = 24576
n-gpu-layers = 99
tensor-split = 1,1
flash-attn = on
parallel = 3
threads = 12
batch-size = 1024
ubatch-size = 1024
temp = 0.3
min-p = 0.05
repeat-penalty = 1.1
timeout = 1200