proxy: add support for user defined metadata in model configs (#333)

Changes: 

- add Metadata key to ModelConfig
- include metadata in /v1/models under meta.llamaswap key
- add recursive macro substitution into Metadata
- change macros at global and model level to be any scalar type

Note: 

This is the first mostly AI generated change to llama-swap. See #333 for notes about the workflow and approach to AI going forward.
This commit is contained in:
Benson Wong
2025-10-04 19:56:41 -07:00
committed by GitHub
parent 1f6179110c
commit 70930e4e91
11 changed files with 807 additions and 25 deletions

View File

@@ -282,6 +282,90 @@ func TestProxyManager_ListModelsHandler(t *testing.T) {
assert.Empty(t, expectedModels, "not all expected models were returned")
}
func TestProxyManager_ListModelsHandler_WithMetadata(t *testing.T) {
// Process config through LoadConfigFromReader to apply macro substitution
configYaml := `
healthCheckTimeout: 15
logLevel: error
startPort: 10000
models:
model1:
cmd: /path/to/server -p ${PORT}
macros:
PORT_NUM: 10001
TEMP: 0.7
NAME: "llama"
metadata:
port: ${PORT_NUM}
temperature: ${TEMP}
enabled: true
note: "Running on port ${PORT_NUM}"
nested:
value: ${TEMP}
model2:
cmd: /path/to/server -p ${PORT}
`
processedConfig, err := config.LoadConfigFromReader(strings.NewReader(configYaml))
assert.NoError(t, err)
proxy := New(processedConfig)
req := httptest.NewRequest("GET", "/v1/models", nil)
w := httptest.NewRecorder()
proxy.ServeHTTP(w, req)
assert.Equal(t, http.StatusOK, w.Code)
var response struct {
Data []map[string]any `json:"data"`
}
err = json.Unmarshal(w.Body.Bytes(), &response)
assert.NoError(t, err)
assert.Len(t, response.Data, 2)
// Find model1 and model2 in response
var model1Data, model2Data map[string]any
for _, model := range response.Data {
if model["id"] == "model1" {
model1Data = model
} else if model["id"] == "model2" {
model2Data = model
}
}
// Verify model1 has llamaswap_meta
assert.NotNil(t, model1Data)
meta, exists := model1Data["meta"]
if !assert.True(t, exists, "model1 should have meta key") {
t.FailNow()
}
metaMap := meta.(map[string]any)
lsmeta, exists := metaMap["llamaswap"]
if !assert.True(t, exists, "model1 should have meta.llamaswap key") {
t.FailNow()
}
lsmetamap := lsmeta.(map[string]any)
// Verify type preservation
assert.Equal(t, float64(10001), lsmetamap["port"]) // JSON numbers are float64
assert.Equal(t, 0.7, lsmetamap["temperature"])
assert.Equal(t, true, lsmetamap["enabled"])
// Verify string interpolation
assert.Equal(t, "Running on port 10001", lsmetamap["note"])
// Verify nested structure
nested := lsmetamap["nested"].(map[string]any)
assert.Equal(t, 0.7, nested["value"])
// Verify model2 does NOT have llamaswap_meta
assert.NotNil(t, model2Data)
_, exists = model2Data["llamaswap_meta"]
assert.False(t, exists, "model2 should not have llamaswap_meta")
}
func TestProxyManager_ListModelsHandler_SortedByID(t *testing.T) {
// Intentionally add models in non-sorted order and with an unlisted model
config := config.Config{