From f20f2c9b7a668251e9b5ff74d4b56d837b6b5692 Mon Sep 17 00:00:00 2001 From: Benson Wong Date: Fri, 14 Feb 2025 12:20:07 -0800 Subject: [PATCH] add docs and container build improvements #43 --- .github/workflows/containers.yml | 14 +++--- README.md | 80 ++++++++++++++++++++++++++------ docker/config.example.yaml | 17 +++++++ docker/llama-swap.Containerfile | 5 +- 4 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 docker/config.example.yaml diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 4ac6a30..32a5b37 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -1,12 +1,14 @@ name: Build Containers on: - # schedule: - # - cron: "0 11 * * *" # Runs daily at 11 AM UTC (3 AM PST) - # push: - # tags: - # - "*" # Triggers on any new tag - workflow_dispatch: # Allows manual triggering of the workflow + # time has no specific meaning, trying to time it after + # the llama.cpp daily packages are published + # https://github.com/ggerganov/llama.cpp/blob/master/.github/workflows/docker.yml + schedule: + - cron: "37 5 * * *" + + # Allows manual triggering of the workflow + workflow_dispatch: jobs: build-and-push: diff --git a/README.md b/README.md index 80d11dc..1b13e30 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,11 @@ ![llama-swap header image](header.jpeg) # llama-swap + llama-swap is a light weight, transparent proxy server that provides automatic model swapping to llama.cpp's server. Written in golang, it is very easy to install (single binary with no dependancies) and configure (single yaml file). -Download a pre-built [release](https://github.com/mostlygeek/llama-swap/releases) or build it yourself from source with `make clean all`. - -## How does it work? -When a request is made to an OpenAI compatible endpoint, lama-swap will extract the `model` value and load the appropriate server configuration to serve it. If a server is already running it will stop it and start the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request. - -In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `profiles` feature can load multiple models at the same time. You have complete control over how your system resources are used. - -## Do I need to use llama.cpp's server (llama-server)? -Any OpenAI compatible server would work. llama-swap was originally designed for llama-server and it is the best supported. - -For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman or docker. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals to shutdown. - ## Features: - ✅ Easy to deploy: single binary with no dependencies @@ -37,6 +26,66 @@ For Python based inference servers like vllm or tabbyAPI it is recommended to ru - ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabbyAPI, etc) - ✅ Direct access to upstream HTTP server via `/upstream/:model_id` ([demo](https://github.com/mostlygeek/llama-swap/pull/31)) +## Docker Install ([download images](https://github.com/mostlygeek/llama-swap/pkgs/container/llama-swap)) + +Docker is the quickest way to try out llama-swap: + +``` +$ docker run -it --rm --runtime nvidia -p 9292:8080 ghcr.io/mostlygeek/llama-swap:cuda + + +# qwen2.5 0.5B +$ curl -s http://localhost:9292/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer no-key" \ + -d '{"model":"qwen2.5","messages": [{"role": "user","content": "tell me a joke"}]}' | \ + jq -r '.choices[0].message.content' + + +# SmolLM2 135M +$ curl -s http://localhost:9292/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer no-key" \ + -d '{"model":"smollm2","messages": [{"role": "user","content": "tell me a joke"}]}' | \ + jq -r '.choices[0].message.content' +``` + +Docker images are [published nightly](https://github.com/mostlygeek/llama-swap/pkgs/container/llama-swap) that include the latest llama-swap and llama-server: + +- `ghcr.io/mostlygeek/llama-swap:cuda` +- `ghcr.io/mostlygeek/llama-swap:intel` +- `ghcr.io/mostlygeek/llama-swap:vulkan` +- `ghcr.io/mostlygeek/llama-swap:musa` + +Specific versions are also available and are tagged with the llama-swap, architecture and llama.cpp versions. For example: `ghcr.io/mostlygeek/llama-swap:v89-cuda-b4716` + +Beyond the demo you will likely want to run the containers with your downloaded models and custom configuration. + +``` +$ docker run -it --rm --runtime nvidia -p 9292:8080 \ + -v /path/to/models:/models \ + -v /path/to/custom/config.yaml:/app/config.yaml \ + ghcr.io/mostlygeek/llama-swap:cuda +``` + +## Bare metal Install ([download](https://github.com/mostlygeek/llama-swap/releases)) + +Pre-built binaries are available for Linux, FreeBSD and Darwin (OSX). These are automatically published and are likely a few hours ahead of the docker releases. The baremetal install works with any OpenAI compatible server, not just llama-server. + +You can also build llama-swap yourself from source with `make clean all`. + +## How does llama-swap work? + +When a request is made to an OpenAI compatible endpoint, lama-swap will extract the `model` value and load the appropriate server configuration to serve it. If a server is already running it will stop it and start the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request. + +In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `profiles` feature can load multiple models at the same time. You have complete control over how your system resources are used. + +## Do I need to use llama.cpp's server (llama-server)? + +Any OpenAI compatible server would work. llama-swap was originally designed for llama-server and it is the best supported. + +For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman or docker. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals to shutdown. + ## config.yaml llama-swap's configuration is purposefully simple. @@ -59,8 +108,8 @@ models: # aliases names to use this model for aliases: - - "gpt-4o-mini" - - "gpt-3.5-turbo" + - "gpt-4o-mini" + - "gpt-3.5-turbo" # check this path for an HTTP 200 OK before serving requests # default: /health to match llama.cpp @@ -121,7 +170,7 @@ profiles: 1. Create a configuration file, see [config.example.yaml](config.example.yaml) 1. Download a [release](https://github.com/mostlygeek/llama-swap/releases) appropriate for your OS and architecture. - * _Note: Windows currently untested._ + - _Note: Windows currently untested._ 1. Run the binary with `llama-swap --config path/to/config.yaml` ### Building from source @@ -156,6 +205,7 @@ curl -Ns 'http://host/logs/stream?no-history' Use this unit file to start llama-swap on boot. This is only tested on Ubuntu. `/etc/systemd/system/llama-swap.service` + ``` [Unit] Description=llama-swap diff --git a/docker/config.example.yaml b/docker/config.example.yaml new file mode 100644 index 0000000..b1ee018 --- /dev/null +++ b/docker/config.example.yaml @@ -0,0 +1,17 @@ +healthCheckTimeout: 300 +logRequests: true + +models: + "qwen2.5": + proxy: "http://127.0.0.1:9999" + cmd: > + /app/llama-server + -hf bartowski/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M + --port 9999 + + "smollm2": + proxy: "http://127.0.0.1:9999" + cmd: > + /app/llama-server + -hf bartowski/SmolLM2-135M-Instruct-GGUF:Q4_K_M + --port 9999 \ No newline at end of file diff --git a/docker/llama-swap.Containerfile b/docker/llama-swap.Containerfile index 583c989..be8a2e4 100644 --- a/docker/llama-swap.Containerfile +++ b/docker/llama-swap.Containerfile @@ -5,11 +5,12 @@ FROM ghcr.io/ggerganov/llama.cpp:${BASE_TAG} ARG LS_VER=89 WORKDIR /app - RUN \ curl -LO https://github.com/mostlygeek/llama-swap/releases/download/v"${LS_VER}"/llama-swap_"${LS_VER}"_linux_amd64.tar.gz && \ tar -zxf llama-swap_"${LS_VER}"_linux_amd64.tar.gz && \ rm llama-swap_"${LS_VER}"_linux_amd64.tar.gz +COPY config.example.yaml /app/config.yaml -ENTRYPOINT [ "/app/llama-swap", "--config", "/config.yaml" ] \ No newline at end of file +HEALTHCHECK CMD curl -f http://localhost:8080/ || exit 1 +ENTRYPOINT [ "/app/llama-swap", "-config", "/app/config.yaml" ] \ No newline at end of file