increase default healthCheckTimeout to 120s

README.md

@@ -63,9 +63,10 @@ models:
 <summary>But also very powerful ...</summary>
 
 ```yaml
-# Seconds to wait for llama.cpp to load and be ready to serve requests
+# Seconds to wait for upstream to load and be ready to serve requests
-# Default (and minimum) is 15 seconds
+# minimum is 15 seconds
-healthCheckTimeout: 60
+# default is 120 seconds
+healthCheckTimeout: 500
 
 # Valid log levels: debug, info (default), warn, error
 logLevel: info

proxy/config.go

@@ -113,7 +113,12 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 		return Config{}, err
 	}
 
-	if config.HealthCheckTimeout < 15 {
+	if config.HealthCheckTimeout == 0 {
+		// this high default timeout helps avoid failing health checks
+		// for configurations that wait for docker or have slower startup
+		config.HealthCheckTimeout = 120
+	} else if config.HealthCheckTimeout < 15 {
+		// set a minimum of 15 seconds
 		config.HealthCheckTimeout = 15
 	}
 

proxy/process.go

@@ -100,7 +100,7 @@ func NewProcess(ID string, healthCheckTimeout int, config ModelConfig, processLo
 		concurrencyLimitSemaphore: make(chan struct{}, concurrentLimit),
 
 		// stop timeout
-		gracefulStopTimeout:        5 * time.Second,
+		gracefulStopTimeout:        10 * time.Second,
 		upstreamWasStoppedWithKill: false,
 	}
 }