Update src/lib/multimodal-live-client.ts

src/lib/multimodal-live-client.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Content, GenerativeContentBlob, Part, InlineDataPart } from "@google/generative-ai";
+import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
 import { EventEmitter } from "eventemitter3";
 import { difference } from "lodash";
 import {
@@ -39,37 +39,12 @@ import {
 } from "../multimodal-live-types";
 import { blobToJSON, base64ToArrayBuffer } from "./utils";
 
-function arrayBufferToBase64(buffer: ArrayBuffer): string {
-    let binary = '';
-    const bytes = new Uint8Array(buffer);
-    const len = bytes.byteLength;
-    for (let i = 0; i < len; i++) {
-        binary += String.fromCharCode(bytes[i]);
-    }
-    if (typeof btoa === 'function') {
-        return btoa(binary);
-    } else if (typeof Buffer !== 'undefined') {
-        return Buffer.from(buffer).toString('base64');
-    } else {
-        throw new Error("Cannot convert ArrayBuffer to Base64 in this environment.");
-    }
-}
-
-function isInlineDataPart(part: Part): part is InlineDataPart {
-    return (
-        typeof part === 'object' &&
-        part !== null &&
-        'inlineData' in part &&
-        typeof (part as any).inlineData === 'object' &&
-        (part as any).inlineData !== null &&
-        typeof (part as any).inlineData.mimeType === 'string' &&
-        typeof (part as any).inlineData.data === 'string'
-    );
-}
-
+/**
+ * the events that this client will emit
+ */
 interface MultimodalLiveClientEventTypes {
   open: () => void;
-  log: (log: StreamingLog) => void; // این هنوز وجود دارد تا side-panel کار کند
+  log: (log: StreamingLog) => void;
   close: (event: CloseEvent) => void;
   audio: (data: ArrayBuffer) => void;
   content: (data: ServerContent) => void;
@@ -85,95 +60,78 @@ export type MultimodalLiveAPIClientConnection = {
   apiKey?: string;
 };
 
+/**
+ * A event-emitting class that manages the connection to the websocket and emits
+ * events to the rest of the application.
+ * If you dont want to use react you can still use this.
+ */
 export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEventTypes> {
   public ws: WebSocket | null = null;
   protected config: LiveConfig | null = null;
   public url: string;
-  private readonly AUDIO_SAMPLE_RATE = 16000;
-  private readonly AUDIO_MIME_TYPE_BASE = `audio/l16`;
-  private readonly AUDIO_MIME_TYPE_WITH_RATE = `${this.AUDIO_MIME_TYPE_BASE};rate=${this.AUDIO_SAMPLE_RATE}`;
-
-  // --- 👇 لاگر داخلی برای ارسال به پنل لاگ، بدون console.log زیاد 👇 ---
-  private logger: ((type: string, message: StreamingLog["message"], count?: number) => void) | null = null;
 
   constructor({ url, apiKey }: MultimodalLiveAPIClientConnection = {}) {
     super();
-    console.log('🔧 Initializing MultimodalLiveClient...'); // فقط لاگ اولیه
+    console.log('🔧 Initializing MultimodalLiveClient with URL:', url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`);
     this.url = url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
-    // اتصال logger داخلی به emit کردن رویداد log
-    this.logger = (type, message, count) => {
-        const logEntry: StreamingLog = { date: new Date(), type, message, count };
-        this.emit("log", logEntry);
-    };
+    this.send = this.send.bind(this);
   }
 
-  // تابع log عمومی حذف شد، از logger داخلی استفاده می‌شود
+  log(type: string, message: StreamingLog["message"]) {
+    const log: StreamingLog = {
+      date: new Date(),
+      type,
+      message,
+    };
+    this.emit("log", log);
+  }
 
   connect(config: LiveConfig): Promise<boolean> {
-    this.logger?.("info", "Attempting WebSocket connection...", undefined);
+    console.log('🔌 Attempting WebSocket connection to:', this.url);
     this.config = config;
-
-    if (this.ws) {
-        this.disconnect();
-    }
+    console.log('🔗 MultimodalLiveClient: Starting WebSocket connection to:', this.url);
 
     const ws = new WebSocket(this.url);
-    ws.binaryType = "arraybuffer";
 
     ws.addEventListener("message", async (evt: MessageEvent) => {
-      if (evt.data instanceof ArrayBuffer || evt.data instanceof Blob) {
-        try {
-            const messageText = await (evt.data instanceof Blob ? evt.data.text() : new TextDecoder().decode(evt.data));
-            const response: LiveIncomingMessage = JSON.parse(messageText);
-            this.receiveParsed(response);
-        } catch (e) {
-             console.error("Error parsing received binary message:", e);
-             this.logger?.("error", "Error parsing received binary message", undefined);
-        }
+      console.log('📨 Received WebSocket message:', evt.data instanceof Blob ? 'Blob data' : evt.data);
+      if (evt.data instanceof Blob) {
+        console.log('📩 MultimodalLiveClient: Received blob message');
+        this.receive(evt.data);
       } else {
-         try {
-            const response: LiveIncomingMessage = JSON.parse(evt.data);
-            this.receiveParsed(response);
-         } catch(e) {
-            console.error("Error parsing received text message:", e, evt.data);
-            this.logger?.("error", `Error parsing text message: ${evt.data ? evt.data.substring(0, 100) + '...' : 'empty'}`, undefined);
-         }
+        console.log("non blob message", evt);
       }
     });
-
     return new Promise((resolve, reject) => {
       const onError = (ev: Event) => {
+        this.disconnect(ws);
         const message = `Could not connect to "${this.url}"`;
-        console.error("WebSocket connection error:", message, ev);
-        this.logger?.(`error.connect`, message, undefined);
+        this.log(`server.${ev.type}`, message);
         reject(new Error(message));
       };
       ws.addEventListener("error", onError);
-
       ws.addEventListener("open", (ev: Event) => {
-        this.logger?.(`client.${ev.type}`, `connected to socket`, undefined);
+        console.log('✅ WebSocket connection opened successfully');
+        if (!this.config) {
+          reject("Invalid config sent to `connect(config)`");
+          return;
+        }
+        console.log('✨ MultimodalLiveClient: WebSocket connection established');
+        this.log(`client.${ev.type}`, `connected to socket`);
         this.emit("open");
 
         this.ws = ws;
 
-        if (!this.config) { // این چک باید بماند
-           console.error("❌ Config not set when WebSocket opened!");
-           this.logger?.("error", "Config not set when WebSocket opened!", undefined);
-           reject("Invalid config state during WebSocket open");
-           return;
-        }
-
         const setupMessage: SetupMessage = {
           setup: this.config,
         };
         this._sendDirect(setupMessage);
-        this.logger?.("client.send.setup", setupMessage, undefined); // ارسال به پنل لاگ
-
-        // --- پیام‌های تحریک کننده و صدای سکوت حذف شده‌اند ---
+        this.log("client.send", "setup");
 
         ws.removeEventListener("error", onError);
-
         ws.addEventListener("close", (ev: CloseEvent) => {
+          console.log(ev);
+          this.disconnect(ws);
           let reason = ev.reason || "";
           if (reason.toLowerCase().includes("error")) {
             const prelude = "ERROR]";
@@ -185,12 +143,11 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
               );
             }
           }
-          this.logger?.(
-            `server.close`,
-            `disconnected ${reason ? `with reason: ${reason}` : `(code: ${ev.code})`}`,
-             ev.code
+          console.log('📝 Close reason:', reason || 'No reason provided');
+          this.log(
+            `server.${ev.type}`,
+            `disconnected ${reason ? `with reason: ${reason}` : ``}`,
           );
-          this.disconnect(ws);
           this.emit("close", ev);
         });
         resolve(true);
@@ -199,137 +156,173 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   }
 
   disconnect(ws?: WebSocket) {
-    const wsToClose = ws || this.ws;
-    if (wsToClose && wsToClose.readyState !== WebSocket.CLOSED && wsToClose.readyState !== WebSocket.CLOSING) {
-      this.logger?.("info", `Closing WebSocket connection (readyState: ${wsToClose.readyState})`, undefined);
-      wsToClose.close();
-      if (this.ws === wsToClose) {
-          this.ws = null;
-      }
+    console.log('🔌 Attempting to disconnect WebSocket');
+    // could be that this is an old websocket and theres already a new instance
+    // only close it if its still the correct reference
+    if ((!ws || this.ws === ws) && this.ws) {
+      console.log('🔒 Closing WebSocket connection');
+      this.ws.close();
+      this.ws = null;
+      this.log("client.close", `Disconnected`);
       return true;
     }
+    console.log('⚠️ No active WebSocket to disconnect');
     return false;
   }
 
-  private createSilentAudioChunk(durationMs: number): Part {
-      // این تابع دیگر استفاده نمی‌شود اما برای کامل بودن نگه داشته شده
-    const bytesPerSample = 2;
-    const numberOfSamples = Math.floor(this.AUDIO_SAMPLE_RATE * (durationMs / 1000));
-    const bufferSize = numberOfSamples * bytesPerSample;
-    const buffer = new ArrayBuffer(bufferSize);
-    const base64Data = arrayBufferToBase64(buffer);
-    return {
-      inlineData: {
-        mimeType: this.AUDIO_MIME_TYPE_WITH_RATE,
-        data: base64Data,
-      }
-    };
-  }
-
-  protected receiveParsed(response: LiveIncomingMessage) {
-     // --- 👇 لاگ دریافت پیام حذف شد 👇 ---
-     // this.logger?.("server.receive", response);
-
+  protected async receive(blob: Blob) {
+    const response: LiveIncomingMessage = (await blobToJSON(
+      blob,
+    )) as LiveIncomingMessage;
+    console.log('📥 Received message:', response);
     if (isToolCallMessage(response)) {
-      this.logger?.("server.toolCall", response, undefined);
+      console.log('🛠️ MultimodalLiveClient: Received tool call');
+      this.log("server.toolCall", response);
       this.emit("toolcall", response.toolCall);
       return;
     }
     if (isToolCallCancellationMessage(response)) {
-      this.logger?.("server.toolCallCancellation", response, undefined);
+      console.log('🚫 MultimodalLiveClient: Received tool call cancellation');
+      this.log("receive.toolCallCancellation", response);
       this.emit("toolcallcancellation", response.toolCallCancellation);
       return;
     }
+
     if (isSetupCompleteMessage(response)) {
-      this.logger?.("server.setupComplete", response, undefined);
+      console.log('🎉 MultimodalLiveClient: Setup complete received');
+      this.log("server.send", "setupComplete");
       this.emit("setupcomplete");
       return;
     }
+
+    // this json also might be `contentUpdate { interrupted: true }`
+    // or contentUpdate { end_of_turn: true }
     if (isServerContentMessage(response)) {
       const { serverContent } = response;
       if (isInterrupted(serverContent)) {
-        this.logger?.("server.interrupted", response, undefined);
+        this.log("receive.serverContent", "interrupted");
         this.emit("interrupted");
         return;
       }
       if (isTurnComplete(serverContent)) {
-        this.logger?.("server.turnComplete", response, undefined);
+        this.log("server.send", "turnComplete");
         this.emit("turncomplete");
+        //plausible theres more to the message, continue
       }
+
       if (isModelTurn(serverContent)) {
         let parts: Part[] = serverContent.modelTurn.parts;
-        const audioParts = parts.filter(isInlineDataPart)
-                                .filter(part => part.inlineData.mimeType.startsWith(this.AUDIO_MIME_TYPE_BASE));
+
+        // when its audio that is returned for modelTurn
+        const audioParts = parts.filter(
+          (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/pcm"),
+        );
+        const base64s = audioParts.map((p) => p.inlineData?.data);
+
+        // strip the audio parts out of the modelTurn
         const otherParts = difference(parts, audioParts);
-        let audioByteLength = 0;
-        audioParts.forEach((part) => {
-          if (part.inlineData.data) {
-            try {
-              const data = base64ToArrayBuffer(part.inlineData.data);
-              this.emit("audio", data); // رویداد صدا هنوز emit می‌شود
-              audioByteLength += data.byteLength;
-            } catch (e) {
-               console.error("Error decoding base64 audio:", e);
-               this.logger?.("error", "Error decoding base64 audio", undefined);
-            }
+        // console.log("otherParts", otherParts);
+
+        base64s.forEach((b64) => {
+          if (b64) {
+            const data = base64ToArrayBuffer(b64);
+            this.emit("audio", data);
+            this.log(`server.audio`, `buffer (${data.byteLength})`);
           }
         });
-        // --- 👇 لاگ بایت صدا حذف شد 👇 ---
-        // if (audioByteLength > 0) {
-        //     this.logger?.(`server.audio`, `buffer`, audioByteLength);
-        // }
-        if (!otherParts.length) return;
+        if (!otherParts.length) {
+          return;
+        }
 
         parts = otherParts;
-        const modelTurnContent: ModelTurn = { modelTurn: { parts } };
-        this.emit("content", modelTurnContent);
-        // --- 👇 لاگ محتوای غیرصوتی حذف شد 👇 ---
-        // this.logger?.(`server.content`, response);
+
+        const content: ModelTurn = { modelTurn: { parts } };
+        this.emit("content", content);
+        this.log(`server.content`, response);
       }
     } else {
-      // لاگ پیام ناشناخته را نگه می‌داریم
-      console.log("Received unrecognized message structure:", response);
-      this.logger?.("server.unknown", response, undefined);
+      console.log("received unmatched message", response);
     }
   }
 
+  /**
+   * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
+   */
   sendRealtimeInput(chunks: GenerativeContentBlob[]) {
-    // --- 👇 لاگ ارسال صدا حذف شد 👇 ---
+    let hasAudio = false;
+    let hasVideo = false;
+    for (let i = 0; i < chunks.length; i++) {
+      const ch = chunks[i];
+      if (ch.mimeType.includes("audio")) {
+        hasAudio = true;
+      }
+      if (ch.mimeType.includes("image")) {
+        hasVideo = true;
+      }
+      if (hasAudio && hasVideo) {
+        break;
+      }
+    }
+    const message =
+      hasAudio && hasVideo
+        ? "audio + video"
+        : hasAudio
+          ? "audio"
+          : hasVideo
+            ? "video"
+            : "unknown";
+
     const data: RealtimeInputMessage = {
-      realtimeInput: { mediaChunks: chunks },
+      realtimeInput: {
+        mediaChunks: chunks,
+      },
     };
     this._sendDirect(data);
-    // this.logger?.(`client.realtimeInput`, `audio/video chunks`, chunks.length);
+    this.log(`client.realtimeInput`, message);
   }
 
+  /**
+   *  send a response to a function call and provide the id of the functions you are responding to
+   */
   sendToolResponse(toolResponse: ToolResponseMessage["toolResponse"]) {
-    const message: ToolResponseMessage = { toolResponse };
+    const message: ToolResponseMessage = {
+      toolResponse,
+    };
+
     this._sendDirect(message);
-    this.logger?.(`client.toolResponse`, message, undefined); // این لاگ معمولا کم تکرار است
+    this.log(`client.toolResponse`, message);
   }
 
+  /**
+   * send normal content parts such as { text }
+   */
   send(parts: Part | Part[], turnComplete: boolean = true) {
     parts = Array.isArray(parts) ? parts : [parts];
-    const content: Content = { role: "user", parts };
+    const content: Content = {
+      role: "user",
+      parts,
+    };
+
     const clientContentRequest: ClientContentMessage = {
-      clientContent: { turns: [content], turnComplete },
+      clientContent: {
+        turns: [content],
+        turnComplete,
+      },
     };
+
     this._sendDirect(clientContentRequest);
-    this.logger?.(`client.send.content`, clientContentRequest, undefined); // این لاگ هم معمولا کم تکرار است
+    this.log(`client.send`, clientContentRequest);
   }
 
+  /**
+   *  used internally to send all messages
+   *  don't use directly unless trying to send an unsupported message type
+   */
   _sendDirect(request: object) {
-    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
-      console.error("WebSocket is not connected or not open. Cannot send message:", request);
-      this.logger?.("error", "Attempted to send message while WebSocket not open", undefined);
-      return;
-    }
-    try {
-      const str = JSON.stringify(request);
-      this.ws.send(str);
-    } catch (error) {
-        console.error("Error stringifying or sending message:", error, request);
-        this.logger?.("error", `Error sending message: ${error}`, undefined);
+    if (!this.ws) {
+      throw new Error("WebSocket is not connected");
     }
+    const str = JSON.stringify(request);
+    this.ws.send(str);
   }
-}
+}