Add cmd_stop configuration to better support docker (#35)
Add `cmd_stop` to model configuration to run a command instead of sending a SIGTERM to shutdown a process before swapping.
This commit is contained in:
@@ -53,6 +53,21 @@ models:
|
||||
--ctx-size 8192
|
||||
--reranking
|
||||
|
||||
# EXPERIMENTAL! Docker Support
|
||||
# see:
|
||||
# - https://github.com/mostlygeek/llama-swap/pull/40
|
||||
# - https://github.com/mostlygeek/llama-swap/issues/35
|
||||
"dockertest":
|
||||
proxy: "http://127.0.0.1:9790"
|
||||
|
||||
# use this to reliably stop named containers
|
||||
cmd_stop: docker stop -t 2 dockertest
|
||||
|
||||
cmd: >
|
||||
docker run --name dockertest
|
||||
--init --rm -p 9790:8080 -v /mnt/nvme/models:/models
|
||||
ghcr.io/ggerganov/llama.cpp:server
|
||||
--model '/models/Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf'
|
||||
|
||||
"simple":
|
||||
# example of setting environment variables
|
||||
|
||||
Reference in New Issue
Block a user