73ad85ea69
Refactor code to support starting of multiple back end llama.cpp servers. This functionality is exposed as `profiles` to create a simple configuration format. Changes: * refactor proxy tests to get ready for multi-process support * update proxy/ProxyManager to support multiple processes (#7) * Add support for Groups in configuration * improve handling of Model alias configs * implement multi-model swapping * improve code clarity for swapModel * improve docs, rename groups to profiles in config
59 lines
1.3 KiB
Go
59 lines
1.3 KiB
Go
package proxy
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
var (
|
|
nextTestPort int = 12000
|
|
portMutex sync.Mutex
|
|
)
|
|
|
|
// Check if the binary exists
|
|
func TestMain(m *testing.M) {
|
|
binaryPath := getSimpleResponderPath()
|
|
if _, err := os.Stat(binaryPath); os.IsNotExist(err) {
|
|
fmt.Printf("simple-responder not found at %s, did you `make simple-responder`?\n", binaryPath)
|
|
os.Exit(1)
|
|
}
|
|
|
|
gin.SetMode(gin.TestMode)
|
|
|
|
m.Run()
|
|
}
|
|
|
|
// Helper function to get the binary path
|
|
func getSimpleResponderPath() string {
|
|
goos := runtime.GOOS
|
|
goarch := runtime.GOARCH
|
|
return filepath.Join("..", "build", fmt.Sprintf("simple-responder_%s_%s", goos, goarch))
|
|
}
|
|
|
|
func getTestSimpleResponderConfig(expectedMessage string) ModelConfig {
|
|
portMutex.Lock()
|
|
defer portMutex.Unlock()
|
|
|
|
port := nextTestPort
|
|
nextTestPort++
|
|
|
|
return getTestSimpleResponderConfigPort(expectedMessage, port)
|
|
}
|
|
|
|
func getTestSimpleResponderConfigPort(expectedMessage string, port int) ModelConfig {
|
|
binaryPath := getSimpleResponderPath()
|
|
|
|
// Create a process configuration
|
|
return ModelConfig{
|
|
Cmd: fmt.Sprintf("%s --port %d --respond '%s'", binaryPath, port, expectedMessage),
|
|
Proxy: fmt.Sprintf("http://127.0.0.1:%d", port),
|
|
CheckEndpoint: "/health",
|
|
}
|
|
}
|