Add Event Bus (#184)

Major internal refactor to use an event bus to pass event/messages along. These changes are largely invisible user facing but sets up internal design for real time stats and information.

- `--watch-config` logic refactored for events
- remove multiple SSE api endpoints, replaced with /api/events
- keep all functionality essentially the same
- UI/backend sync is in near real time now

ui/src/contexts/APIProvider.tsx

@@ -6,6 +6,8 @@ const LOG_LENGTH_LIMIT = 1024 * 100; /* 100KB of log data */
 export interface Model {
   id: string;
   state: ModelStatus;
+  name: string;
+  description: string;
 }
 
 interface APIProviderType {
@@ -13,12 +15,18 @@ interface APIProviderType {
   listModels: () => Promise<Model[]>;
   unloadAllModels: () => Promise<void>;
   loadModel: (model: string) => Promise<void>;
-  enableProxyLogs: (enabled: boolean) => void;
-  enableUpstreamLogs: (enabled: boolean) => void;
-  enableModelUpdates: (enabled: boolean) => void;
+  enableAPIEvents: (enabled: boolean) => void;
   proxyLogs: string;
   upstreamLogs: string;
 }
+interface LogData {
+  source: "upstream" | "proxy";
+  data: string;
+}
+interface APIEventEnvelope {
+  type: "modelStatus" | "logData";
+  data: string;
+}
 
 const APIContext = createContext<APIProviderType | undefined>(undefined);
 type APIProviderProps = {
@@ -30,6 +38,7 @@ export function APIProvider({ children }: APIProviderProps) {
   const [upstreamLogs, setUpstreamLogs] = useState("");
   const proxyEventSource = useRef<EventSource | null>(null);
   const upstreamEventSource = useRef<EventSource | null>(null);
+  const apiEventSource = useRef<EventSource | null>(null);
 
   const [models, setModels] = useState<Model[]>([]);
   const modelStatusEventSource = useRef<EventSource | null>(null);
@@ -41,104 +50,61 @@ export function APIProvider({ children }: APIProviderProps) {
     });
   }, []);
 
-  const handleProxyMessage = useCallback(
-    (e: MessageEvent) => {
-      appendLog(e.data, setProxyLogs);
-    },
-    [proxyLogs, appendLog]
-  );
+  const enableAPIEvents = useCallback((enabled: boolean) => {
+    if (!enabled) {
+      apiEventSource.current?.close();
+      apiEventSource.current = null;
+      return;
+    }
 
-  const handleUpstreamMessage = useCallback(
-    (e: MessageEvent) => {
-      appendLog(e.data, setUpstreamLogs);
-    },
-    [appendLog]
-  );
+    let retryCount = 0;
+    const maxRetries = 3;
+    const initialDelay = 1000; // 1 second
 
-  const enableProxyLogs = useCallback(
-    (enabled: boolean) => {
-      if (enabled) {
-        let retryCount = 0;
-        const maxRetries = 3;
-        const initialDelay = 1000; // 1 second
+    const connect = () => {
+      const eventSource = new EventSource("/api/events");
 
-        const connect = () => {
-          const eventSource = new EventSource("/logs/streamSSE/proxy");
+      eventSource.onmessage = (e: MessageEvent) => {
+        try {
+          const message = JSON.parse(e.data) as APIEventEnvelope;
+          switch (message.type) {
+            case "modelStatus":
+              {
+                const models = JSON.parse(message.data) as Model[];
+                setModels(models);
+              }
+              break;
 
-          eventSource.onmessage = handleProxyMessage;
-          eventSource.onerror = () => {
-            eventSource.close();
-            if (retryCount < maxRetries) {
-              retryCount++;
-              const delay = initialDelay * Math.pow(2, retryCount - 1);
-              setTimeout(connect, delay);
+            case "logData": {
+              const logData = JSON.parse(message.data) as LogData;
+              switch (logData.source) {
+                case "proxy":
+                  appendLog(logData.data, setProxyLogs);
+                  break;
+                case "upstream":
+                  appendLog(logData.data, setUpstreamLogs);
+                  break;
+              }
             }
-          };
-
-          proxyEventSource.current = eventSource;
-        };
-
-        connect();
-      } else {
-        proxyEventSource.current?.close();
-        proxyEventSource.current = null;
-      }
-    },
-    [handleProxyMessage]
-  );
-
-  const enableUpstreamLogs = useCallback(
-    (enabled: boolean) => {
-      if (enabled) {
-        let retryCount = 0;
-        const maxRetries = 3;
-        const initialDelay = 1000; // 1 second
-
-        const connect = () => {
-          const eventSource = new EventSource("/logs/streamSSE/upstream");
-
-          eventSource.onmessage = handleUpstreamMessage;
-          eventSource.onerror = () => {
-            eventSource.close();
-            if (retryCount < maxRetries) {
-              retryCount++;
-              const delay = initialDelay * Math.pow(2, retryCount - 1);
-              setTimeout(connect, delay);
-            }
-          };
-
-          upstreamEventSource.current = eventSource;
-        };
-
-        connect();
-      } else {
-        upstreamEventSource.current?.close();
-        upstreamEventSource.current = null;
-      }
-    },
-    [handleUpstreamMessage]
-  );
-
-  const enableModelUpdates = useCallback(
-    (enabled: boolean) => {
-      if (enabled) {
-        const eventSource = new EventSource("/api/modelsSSE");
-        eventSource.onmessage = (e: MessageEvent) => {
-          try {
-            const models = JSON.parse(e.data) as Model[];
-            setModels(models);
-          } catch (e) {
-            console.error(e);
           }
-        };
-        modelStatusEventSource.current = eventSource;
-      } else {
-        modelStatusEventSource.current?.close();
-        modelStatusEventSource.current = null;
-      }
-    },
-    [setModels]
-  );
+        } catch (err) {
+          console.error(e.data, err);
+        }
+      };
+      eventSource.onerror = () => {
+        eventSource.close();
+        if (retryCount < maxRetries) {
+          retryCount++;
+          const delay = initialDelay * Math.pow(2, retryCount - 1);
+          setTimeout(connect, delay);
+        }
+      };
+
+      apiEventSource.current = eventSource;
+    };
+
+    connect();
+  }, []);
 
   useEffect(() => {
     return () => {
@@ -196,23 +162,11 @@ export function APIProvider({ children }: APIProviderProps) {
       listModels,
       unloadAllModels,
       loadModel,
-      enableProxyLogs,
-      enableUpstreamLogs,
-      enableModelUpdates,
+      enableAPIEvents,
       proxyLogs,
       upstreamLogs,
     }),
-    [
-      models,
-      listModels,
-      unloadAllModels,
-      loadModel,
-      enableProxyLogs,
-      enableUpstreamLogs,
-      enableModelUpdates,
-      proxyLogs,
-      upstreamLogs,
-    ]
+    [models, listModels, unloadAllModels, loadModel, enableAPIEvents, proxyLogs, upstreamLogs]
   );
 
   return <APIContext.Provider value={value}>{children}</APIContext.Provider>;

ui/src/pages/LogViewer.tsx

@@ -3,14 +3,12 @@ import { useAPI } from "../contexts/APIProvider";
 import { usePersistentState } from "../hooks/usePersistentState";
 
 const LogViewer = () => {
-  const { proxyLogs, upstreamLogs, enableProxyLogs, enableUpstreamLogs } = useAPI();
+  const { proxyLogs, upstreamLogs, enableAPIEvents } = useAPI();
 
   useEffect(() => {
-    enableProxyLogs(true);
-    enableUpstreamLogs(true);
+    enableAPIEvents(true);
     return () => {
-      enableProxyLogs(false);
-      enableUpstreamLogs(false);
+      enableAPIEvents(false);
     };
   }, []);
 

ui/src/pages/Models.tsx

@@ -4,15 +4,13 @@ import { LogPanel } from "./LogViewer";
 import { processEvalTimes } from "../lib/Utils";
 
 export default function ModelsPage() {
-  const { models, enableModelUpdates, unloadAllModels, loadModel, upstreamLogs, enableUpstreamLogs } = useAPI();
+  const { models, unloadAllModels, loadModel, upstreamLogs, enableAPIEvents } = useAPI();
   const [isUnloading, setIsUnloading] = useState(false);
 
   useEffect(() => {
-    enableModelUpdates(true);
-    enableUpstreamLogs(true);
+    enableAPIEvents(true);
     return () => {
-      enableModelUpdates(false);
-      enableUpstreamLogs(false);
+      enableAPIEvents(false);
     };
   }, []);
 
@@ -57,8 +55,13 @@ export default function ModelsPage() {
                   <tr key={model.id} className="border-b hover:bg-secondary-hover border-border">
                     <td className="p-2">
                       <a href={`/upstream/${model.id}/`} className="underline" target="_blank">
-                        {model.id}
+                        {model.name !== "" ? model.name : model.id}
                       </a>
+                      {model.description != "" && (
+                        <p>
+                          <em>{model.description}</em>
+                        </p>
+                      )}
                     </td>
                     <td className="p-2">
                       <button