From 216c40b951f0ae823bee548f3b8f7d7242286693 Mon Sep 17 00:00:00 2001 From: Benson Wong Date: Sun, 28 Sep 2025 16:50:06 -0700 Subject: [PATCH] proxy/config: create config package and migrate configuration (#329) * proxy/config: create config package and migrate configuration The configuration is become more complex as llama-swap adds more advanced features. This commit moves config to its own package so it can be developed independently of the proxy package. Additionally, enforcing a public API for a configuration will allow downstream usage to be more decoupled. --- Makefile | 7 +- llama-swap.go | 13 +-- proxy/{ => config}/config.go | 2 +- proxy/{ => config}/config_posix_test.go | 2 +- proxy/{ => config}/config_test.go | 2 +- proxy/{ => config}/config_windows_test.go | 2 +- proxy/helpers_test.go | 7 +- proxy/metrics_monitor.go | 3 +- proxy/process.go | 7 +- proxy/process_test.go | 23 ++--- proxy/processgroup.go | 6 +- proxy/processgroup_test.go | 15 +-- proxy/proxymanager.go | 5 +- proxy/proxymanager_test.go | 111 +++++++++++----------- 14 files changed, 108 insertions(+), 97 deletions(-) rename proxy/{ => config}/config.go (99%) rename proxy/{ => config}/config_posix_test.go (99%) rename proxy/{ => config}/config_test.go (99%) rename proxy/{ => config}/config_windows_test.go (99%) diff --git a/Makefile b/Makefile index 8e01898..330c434 100644 --- a/Makefile +++ b/Makefile @@ -24,10 +24,11 @@ proxy/ui_dist/placeholder.txt: touch $@ test: proxy/ui_dist/placeholder.txt - go test -short -v -count=1 ./proxy + go test -short -count=1 ./proxy/... +# for CI - full test (takes longer) test-all: proxy/ui_dist/placeholder.txt - go test -v -count=1 ./proxy + go test -count=1 ./proxy/... ui/node_modules: cd ui && npm install @@ -81,4 +82,4 @@ release: git tag "$$new_tag"; # Phony targets -.PHONY: all clean ui mac linux windows simple-responder +.PHONY: all clean ui mac linux windows simple-responder test test-all diff --git a/llama-swap.go b/llama-swap.go index 3bdcef1..bf93a41 100644 --- a/llama-swap.go +++ b/llama-swap.go @@ -16,6 +16,7 @@ import ( "github.com/gin-gonic/gin" "github.com/mostlygeek/llama-swap/event" "github.com/mostlygeek/llama-swap/proxy" + "github.com/mostlygeek/llama-swap/proxy/config" ) var ( @@ -38,13 +39,13 @@ func main() { os.Exit(0) } - config, err := proxy.LoadConfig(*configPath) + conf, err := config.LoadConfig(*configPath) if err != nil { fmt.Printf("Error loading config: %v\n", err) os.Exit(1) } - if len(config.Profiles) > 0 { + if len(conf.Profiles) > 0 { fmt.Println("WARNING: Profile functionality has been removed in favor of Groups. See the README for more information.") } @@ -67,7 +68,7 @@ func main() { // Support for watching config and reloading when it changes reloadProxyManager := func() { if currentPM, ok := srv.Handler.(*proxy.ProxyManager); ok { - config, err = proxy.LoadConfig(*configPath) + conf, err = config.LoadConfig(*configPath) if err != nil { fmt.Printf("Warning, unable to reload configuration: %v\n", err) return @@ -75,7 +76,7 @@ func main() { fmt.Println("Configuration Changed") currentPM.Shutdown() - srv.Handler = proxy.New(config) + srv.Handler = proxy.New(conf) fmt.Println("Configuration Reloaded") // wait a few seconds and tell any UI to reload @@ -85,12 +86,12 @@ func main() { }) }) } else { - config, err = proxy.LoadConfig(*configPath) + conf, err = config.LoadConfig(*configPath) if err != nil { fmt.Printf("Error, unable to load configuration: %v\n", err) os.Exit(1) } - srv.Handler = proxy.New(config) + srv.Handler = proxy.New(conf) } } diff --git a/proxy/config.go b/proxy/config/config.go similarity index 99% rename from proxy/config.go rename to proxy/config/config.go index 65be8d0..1dc8223 100644 --- a/proxy/config.go +++ b/proxy/config/config.go @@ -1,4 +1,4 @@ -package proxy +package config import ( "fmt" diff --git a/proxy/config_posix_test.go b/proxy/config/config_posix_test.go similarity index 99% rename from proxy/config_posix_test.go rename to proxy/config/config_posix_test.go index 122d351..9be46e1 100644 --- a/proxy/config_posix_test.go +++ b/proxy/config/config_posix_test.go @@ -1,6 +1,6 @@ //go:build !windows -package proxy +package config import ( "os" diff --git a/proxy/config_test.go b/proxy/config/config_test.go similarity index 99% rename from proxy/config_test.go rename to proxy/config/config_test.go index 505d80b..0bee014 100644 --- a/proxy/config_test.go +++ b/proxy/config/config_test.go @@ -1,4 +1,4 @@ -package proxy +package config import ( "slices" diff --git a/proxy/config_windows_test.go b/proxy/config/config_windows_test.go similarity index 99% rename from proxy/config_windows_test.go rename to proxy/config/config_windows_test.go index 6902da0..e89bfcf 100644 --- a/proxy/config_windows_test.go +++ b/proxy/config/config_windows_test.go @@ -1,6 +1,6 @@ //go:build windows -package proxy +package config import ( "os" diff --git a/proxy/helpers_test.go b/proxy/helpers_test.go index bcb5acb..95b8e6b 100644 --- a/proxy/helpers_test.go +++ b/proxy/helpers_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/mostlygeek/llama-swap/proxy/config" "gopkg.in/yaml.v3" ) @@ -65,18 +66,18 @@ func getTestPort() int { return port } -func getTestSimpleResponderConfig(expectedMessage string) ModelConfig { +func getTestSimpleResponderConfig(expectedMessage string) config.ModelConfig { return getTestSimpleResponderConfigPort(expectedMessage, getTestPort()) } -func getTestSimpleResponderConfigPort(expectedMessage string, port int) ModelConfig { +func getTestSimpleResponderConfigPort(expectedMessage string, port int) config.ModelConfig { // Create a YAML string with just the values we want to set yamlStr := fmt.Sprintf(` cmd: '%s --port %d --silent --respond %s' proxy: "http://127.0.0.1:%d" `, simpleResponderPath, port, expectedMessage, port) - var cfg ModelConfig + var cfg config.ModelConfig if err := yaml.Unmarshal([]byte(yamlStr), &cfg); err != nil { panic(fmt.Sprintf("failed to unmarshal test config: %v in [%s]", err, yamlStr)) } diff --git a/proxy/metrics_monitor.go b/proxy/metrics_monitor.go index ee11f2a..826870f 100644 --- a/proxy/metrics_monitor.go +++ b/proxy/metrics_monitor.go @@ -6,6 +6,7 @@ import ( "time" "github.com/mostlygeek/llama-swap/event" + "github.com/mostlygeek/llama-swap/proxy/config" ) // TokenMetrics represents parsed token statistics from llama-server logs @@ -38,7 +39,7 @@ type MetricsMonitor struct { nextID int } -func NewMetricsMonitor(config *Config) *MetricsMonitor { +func NewMetricsMonitor(config *config.Config) *MetricsMonitor { maxMetrics := config.MetricsMaxInMemory if maxMetrics <= 0 { maxMetrics = 1000 // Default fallback diff --git a/proxy/process.go b/proxy/process.go index 94c004b..51a5bc6 100644 --- a/proxy/process.go +++ b/proxy/process.go @@ -16,6 +16,7 @@ import ( "time" "github.com/mostlygeek/llama-swap/event" + "github.com/mostlygeek/llama-swap/proxy/config" ) type ProcessState string @@ -39,7 +40,7 @@ const ( type Process struct { ID string - config ModelConfig + config config.ModelConfig cmd *exec.Cmd // PR #155 called to cancel the upstream process @@ -74,7 +75,7 @@ type Process struct { failedStartCount int } -func NewProcess(ID string, healthCheckTimeout int, config ModelConfig, processLogger *LogMonitor, proxyLogger *LogMonitor) *Process { +func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, processLogger *LogMonitor, proxyLogger *LogMonitor) *Process { concurrentLimit := 10 if config.ConcurrencyLimit > 0 { concurrentLimit = config.ConcurrencyLimit @@ -539,7 +540,7 @@ func (p *Process) cmdStopUpstreamProcess() error { if p.config.CmdStop != "" { // replace ${PID} with the pid of the process - stopArgs, err := SanitizeCommand(strings.ReplaceAll(p.config.CmdStop, "${PID}", fmt.Sprintf("%d", p.cmd.Process.Pid))) + stopArgs, err := config.SanitizeCommand(strings.ReplaceAll(p.config.CmdStop, "${PID}", fmt.Sprintf("%d", p.cmd.Process.Pid))) if err != nil { p.proxyLogger.Errorf("<%s> Failed to sanitize stop command: %v", p.ID, err) return err diff --git a/proxy/process_test.go b/proxy/process_test.go index da4a380..574c5d9 100644 --- a/proxy/process_test.go +++ b/proxy/process_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/mostlygeek/llama-swap/proxy/config" "github.com/stretchr/testify/assert" ) @@ -90,7 +91,7 @@ func TestProcess_WaitOnMultipleStarts(t *testing.T) { // test that the automatic start returns the expected error type func TestProcess_BrokenModelConfig(t *testing.T) { // Create a process configuration - config := ModelConfig{ + config := config.ModelConfig{ Cmd: "nonexistent-command", Proxy: "http://127.0.0.1:9913", CheckEndpoint: "/health", @@ -325,7 +326,7 @@ func TestProcess_ExitInterruptsHealthCheck(t *testing.T) { // should run and exit but interrupt the long checkHealthTimeout checkHealthTimeout := 5 - config := ModelConfig{ + config := config.ModelConfig{ Cmd: "sleep 1", Proxy: "http://127.0.0.1:9913", CheckEndpoint: "/health", @@ -402,7 +403,7 @@ func TestProcess_ForceStopWithKill(t *testing.T) { binaryPath := getSimpleResponderPath() port := getTestPort() - config := ModelConfig{ + conf := config.ModelConfig{ // note --ignore-sig-term which ignores the SIGTERM signal so a SIGKILL must be sent // to force the process to exit Cmd: fmt.Sprintf("%s --port %d --respond %s --silent --ignore-sig-term", binaryPath, port, expectedMessage), @@ -410,7 +411,7 @@ func TestProcess_ForceStopWithKill(t *testing.T) { CheckEndpoint: "/health", } - process := NewProcess("stop_immediate", 2, config, debugLogger, debugLogger) + process := NewProcess("stop_immediate", 2, conf, debugLogger, debugLogger) defer process.Stop() // reduce to make testing go faster @@ -450,15 +451,15 @@ func TestProcess_ForceStopWithKill(t *testing.T) { } func TestProcess_StopCmd(t *testing.T) { - config := getTestSimpleResponderConfig("test_stop_cmd") + conf := getTestSimpleResponderConfig("test_stop_cmd") if runtime.GOOS == "windows" { - config.CmdStop = "taskkill /f /t /pid ${PID}" + conf.CmdStop = "taskkill /f /t /pid ${PID}" } else { - config.CmdStop = "kill -TERM ${PID}" + conf.CmdStop = "kill -TERM ${PID}" } - process := NewProcess("testStopCmd", 2, config, debugLogger, debugLogger) + process := NewProcess("testStopCmd", 2, conf, debugLogger, debugLogger) defer process.Stop() err := process.start() @@ -470,15 +471,15 @@ func TestProcess_StopCmd(t *testing.T) { func TestProcess_EnvironmentSetCorrectly(t *testing.T) { expectedMessage := "test_env_not_emptied" - config := getTestSimpleResponderConfig(expectedMessage) + conf := getTestSimpleResponderConfig(expectedMessage) // ensure that the the default config does not blank out the inherited environment - configWEnv := config + configWEnv := conf // ensure the additiona variables are appended to the process' environment configWEnv.Env = append(configWEnv.Env, "TEST_ENV1=1", "TEST_ENV2=2") - process1 := NewProcess("env_test", 2, config, debugLogger, debugLogger) + process1 := NewProcess("env_test", 2, conf, debugLogger, debugLogger) process2 := NewProcess("env_test", 2, configWEnv, debugLogger, debugLogger) process1.start() diff --git a/proxy/processgroup.go b/proxy/processgroup.go index e4ce63e..e0b0600 100644 --- a/proxy/processgroup.go +++ b/proxy/processgroup.go @@ -5,12 +5,14 @@ import ( "net/http" "slices" "sync" + + "github.com/mostlygeek/llama-swap/proxy/config" ) type ProcessGroup struct { sync.Mutex - config Config + config config.Config id string swap bool exclusive bool @@ -24,7 +26,7 @@ type ProcessGroup struct { lastUsedProcess string } -func NewProcessGroup(id string, config Config, proxyLogger *LogMonitor, upstreamLogger *LogMonitor) *ProcessGroup { +func NewProcessGroup(id string, config config.Config, proxyLogger *LogMonitor, upstreamLogger *LogMonitor) *ProcessGroup { groupConfig, ok := config.Groups[id] if !ok { panic("Unable to find configuration for group id: " + id) diff --git a/proxy/processgroup_test.go b/proxy/processgroup_test.go index 791a5a9..f10293a 100644 --- a/proxy/processgroup_test.go +++ b/proxy/processgroup_test.go @@ -7,19 +7,20 @@ import ( "sync" "testing" + "github.com/mostlygeek/llama-swap/proxy/config" "github.com/stretchr/testify/assert" ) -var processGroupTestConfig = AddDefaultGroupToConfig(Config{ +var processGroupTestConfig = config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), "model2": getTestSimpleResponderConfig("model2"), "model3": getTestSimpleResponderConfig("model3"), "model4": getTestSimpleResponderConfig("model4"), "model5": getTestSimpleResponderConfig("model5"), }, - Groups: map[string]GroupConfig{ + Groups: map[string]config.GroupConfig{ "G1": { Swap: true, Exclusive: true, @@ -34,7 +35,7 @@ var processGroupTestConfig = AddDefaultGroupToConfig(Config{ }) func TestProcessGroup_DefaultHasCorrectModel(t *testing.T) { - pg := NewProcessGroup(DEFAULT_GROUP_ID, processGroupTestConfig, testLogger, testLogger) + pg := NewProcessGroup(config.DEFAULT_GROUP_ID, processGroupTestConfig, testLogger, testLogger) assert.True(t, pg.HasMember("model5")) } @@ -48,9 +49,9 @@ func TestProcessGroup_HasMember(t *testing.T) { // TestProcessGroup_ProxyRequestSwapIsTrueParallel tests that when swap is true // and multiple requests are made in parallel, only one process is running at a time. func TestProcessGroup_ProxyRequestSwapIsTrueParallel(t *testing.T) { - var processGroupTestConfig = AddDefaultGroupToConfig(Config{ + var processGroupTestConfig = config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ // use the same listening so if a model is already running, it will fail // this is a way to test that swap isolation is working // properly when there are parallel requests made at the @@ -61,7 +62,7 @@ func TestProcessGroup_ProxyRequestSwapIsTrueParallel(t *testing.T) { "model4": getTestSimpleResponderConfigPort("model4", 9832), "model5": getTestSimpleResponderConfigPort("model5", 9832), }, - Groups: map[string]GroupConfig{ + Groups: map[string]config.GroupConfig{ "G1": { Swap: true, Members: []string{"model1", "model2", "model3", "model4", "model5"}, diff --git a/proxy/proxymanager.go b/proxy/proxymanager.go index e615b03..50b677d 100644 --- a/proxy/proxymanager.go +++ b/proxy/proxymanager.go @@ -16,6 +16,7 @@ import ( "github.com/gin-gonic/gin" "github.com/mostlygeek/llama-swap/event" + "github.com/mostlygeek/llama-swap/proxy/config" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -27,7 +28,7 @@ const ( type ProxyManager struct { sync.Mutex - config Config + config config.Config ginEngine *gin.Engine // logging @@ -44,7 +45,7 @@ type ProxyManager struct { shutdownCancel context.CancelFunc } -func New(config Config) *ProxyManager { +func New(config config.Config) *ProxyManager { // set up loggers stdoutLogger := NewLogMonitorWriter(os.Stdout) upstreamLogger := NewLogMonitorWriter(stdoutLogger) diff --git a/proxy/proxymanager_test.go b/proxy/proxymanager_test.go index 446a095..0a99f9d 100644 --- a/proxy/proxymanager_test.go +++ b/proxy/proxymanager_test.go @@ -16,14 +16,15 @@ import ( "time" "github.com/mostlygeek/llama-swap/event" + "github.com/mostlygeek/llama-swap/proxy/config" "github.com/stretchr/testify/assert" "github.com/tidwall/gjson" ) func TestProxyManager_SwapProcessCorrectly(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), "model2": getTestSimpleResponderConfig("model2"), }, @@ -44,14 +45,14 @@ func TestProxyManager_SwapProcessCorrectly(t *testing.T) { } } func TestProxyManager_SwapMultiProcess(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), "model2": getTestSimpleResponderConfig("model2"), }, LogLevel: "error", - Groups: map[string]GroupConfig{ + Groups: map[string]config.GroupConfig{ "G1": { Swap: true, Exclusive: false, @@ -89,14 +90,14 @@ func TestProxyManager_SwapMultiProcess(t *testing.T) { // Test that a persistent group is not affected by the swapping behaviour of // other groups. func TestProxyManager_PersistentGroupsAreNotSwapped(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), // goes into the default group "model2": getTestSimpleResponderConfig("model2"), }, LogLevel: "error", - Groups: map[string]GroupConfig{ + Groups: map[string]config.GroupConfig{ // the forever group is persistent and should not be affected by model1 "forever": { Swap: true, @@ -133,9 +134,9 @@ func TestProxyManager_SwapMultiProcessParallelRequests(t *testing.T) { t.Skip("skipping slow test") } - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), "model2": getTestSimpleResponderConfig("model2"), "model3": getTestSimpleResponderConfig("model3"), @@ -196,9 +197,9 @@ func TestProxyManager_ListModelsHandler(t *testing.T) { model2Config.Name = " " // empty whitespace only strings will get ignored model2Config.Description = " " - config := Config{ + config := config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": model1Config, "model2": model2Config, "model3": getTestSimpleResponderConfig("model3"), @@ -283,13 +284,13 @@ func TestProxyManager_ListModelsHandler(t *testing.T) { func TestProxyManager_ListModelsHandler_SortedByID(t *testing.T) { // Intentionally add models in non-sorted order and with an unlisted model - config := Config{ + config := config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "zeta": getTestSimpleResponderConfig("zeta"), "alpha": getTestSimpleResponderConfig("alpha"), "beta": getTestSimpleResponderConfig("beta"), - "hidden": func() ModelConfig { + "hidden": func() config.ModelConfig { mc := getTestSimpleResponderConfig("hidden") mc.Unlisted = true return mc @@ -337,15 +338,15 @@ func TestProxyManager_Shutdown(t *testing.T) { model3Config := getTestSimpleResponderConfigPort("model3", 9993) model3Config.Proxy = "http://localhost:10003/" - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": model1Config, "model2": model2Config, "model3": model3Config, }, LogLevel: "error", - Groups: map[string]GroupConfig{ + Groups: map[string]config.GroupConfig{ "test": { Swap: false, Members: []string{"model1", "model2", "model3"}, @@ -380,21 +381,21 @@ func TestProxyManager_Shutdown(t *testing.T) { } func TestProxyManager_Unload(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + conf := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", }) - proxy := New(config) + proxy := New(conf) reqBody := fmt.Sprintf(`{"model":"%s"}`, "model1") req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody)) w := httptest.NewRecorder() proxy.ServeHTTP(w, req) - assert.Equal(t, proxy.processGroups[DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateReady) + assert.Equal(t, proxy.processGroups[config.DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateReady) req = httptest.NewRequest("GET", "/unload", nil) w = httptest.NewRecorder() proxy.ServeHTTP(w, req) @@ -402,23 +403,23 @@ func TestProxyManager_Unload(t *testing.T) { assert.Equal(t, w.Body.String(), "OK") select { - case <-proxy.processGroups[DEFAULT_GROUP_ID].processes["model1"].cmdWaitChan: + case <-proxy.processGroups[config.DEFAULT_GROUP_ID].processes["model1"].cmdWaitChan: // good case <-time.After(2 * time.Second): t.Fatal("timeout waiting for model1 to stop") } - assert.Equal(t, proxy.processGroups[DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateStopped) + assert.Equal(t, proxy.processGroups[config.DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateStopped) } func TestProxyManager_UnloadSingleModel(t *testing.T) { const testGroupId = "testGroup" - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), "model2": getTestSimpleResponderConfig("model2"), }, - Groups: map[string]GroupConfig{ + Groups: map[string]config.GroupConfig{ testGroupId: { Swap: false, Members: []string{"model1", "model2"}, @@ -463,9 +464,9 @@ func TestProxyManager_UnloadSingleModel(t *testing.T) { // Test issue #61 `Listing the current list of models and the loaded model.` func TestProxyManager_RunningEndpoint(t *testing.T) { // Shared configuration - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), "model2": getTestSimpleResponderConfig("model2"), }, @@ -528,9 +529,9 @@ func TestProxyManager_RunningEndpoint(t *testing.T) { } func TestProxyManager_AudioTranscriptionHandler(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "TheExpectedModel": getTestSimpleResponderConfig("TheExpectedModel"), }, LogLevel: "error", @@ -581,15 +582,15 @@ func TestProxyManager_UseModelName(t *testing.T) { modelConfig := getTestSimpleResponderConfig(upstreamModelName) modelConfig.UseModelName = upstreamModelName - config := AddDefaultGroupToConfig(Config{ + conf := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": modelConfig, }, LogLevel: "error", }) - proxy := New(config) + proxy := New(conf) defer proxy.StopProcesses(StopWaitForInflightRequest) requestedModel := "model1" @@ -644,9 +645,9 @@ func TestProxyManager_UseModelName(t *testing.T) { } func TestProxyManager_CORSOptionsHandler(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -720,7 +721,7 @@ models: aliases: [model-alias] `, getSimpleResponderPath()) - config, err := LoadConfigFromReader(strings.NewReader(configStr)) + config, err := config.LoadConfigFromReader(strings.NewReader(configStr)) assert.NoError(t, err) proxy := New(config) @@ -743,9 +744,9 @@ models: } func TestProxyManager_ChatContentLength(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -768,14 +769,14 @@ func TestProxyManager_ChatContentLength(t *testing.T) { func TestProxyManager_FiltersStripParams(t *testing.T) { modelConfig := getTestSimpleResponderConfig("model1") - modelConfig.Filters = ModelFilters{ + modelConfig.Filters = config.ModelFilters{ StripParams: "temperature, model, stream", } - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, LogLevel: "error", - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": modelConfig, }, }) @@ -801,9 +802,9 @@ func TestProxyManager_FiltersStripParams(t *testing.T) { } func TestProxyManager_MiddlewareWritesMetrics_NonStreaming(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -836,9 +837,9 @@ func TestProxyManager_MiddlewareWritesMetrics_NonStreaming(t *testing.T) { } func TestProxyManager_MiddlewareWritesMetrics_Streaming(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -871,9 +872,9 @@ func TestProxyManager_MiddlewareWritesMetrics_Streaming(t *testing.T) { } func TestProxyManager_HealthEndpoint(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -890,9 +891,9 @@ func TestProxyManager_HealthEndpoint(t *testing.T) { // Ensure the custom llama-server /completion endpoint proxies correctly func TestProxyManager_CompletionEndpoint(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -936,7 +937,7 @@ models: `, "${simpleresponderpath}", simpleResponderPath, -1) // Create a test model configuration - config, err := LoadConfigFromReader(strings.NewReader(configStr)) + config, err := config.LoadConfigFromReader(strings.NewReader(configStr)) if !assert.NoError(t, err, "Invalid configuration") { return } @@ -970,9 +971,9 @@ models: } func TestProxyManager_StreamingEndpointsReturnNoBufferingHeader(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "model1": getTestSimpleResponderConfig("model1"), }, LogLevel: "error", @@ -1009,9 +1010,9 @@ func TestProxyManager_StreamingEndpointsReturnNoBufferingHeader(t *testing.T) { } func TestProxyManager_ProxiedStreamingEndpointReturnsNoBufferingHeader(t *testing.T) { - config := AddDefaultGroupToConfig(Config{ + config := config.AddDefaultGroupToConfig(config.Config{ HealthCheckTimeout: 15, - Models: map[string]ModelConfig{ + Models: map[string]config.ModelConfig{ "streaming-model": getTestSimpleResponderConfig("streaming-model"), }, LogLevel: "error",