Update src/lib/multimodal-live-client.ts

src/lib/multimodal-live-client.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Content, GenerativeContentBlob, Part } from "@google/generative-ai";
+import { Content, GenerativeContentBlob, Part, InlineDataPart } from "@google/generative-ai";
 import { EventEmitter } from "eventemitter3";
 import { difference } from "lodash";
 import {
@@ -39,12 +39,37 @@ import {
 } from "../multimodal-live-types";
 import { blobToJSON, base64ToArrayBuffer } from "./utils";
 
-/**
- * the events that this client will emit
- */
+function arrayBufferToBase64(buffer: ArrayBuffer): string {
+    let binary = '';
+    const bytes = new Uint8Array(buffer);
+    const len = bytes.byteLength;
+    for (let i = 0; i < len; i++) {
+        binary += String.fromCharCode(bytes[i]);
+    }
+    if (typeof btoa === 'function') {
+        return btoa(binary);
+    } else if (typeof Buffer !== 'undefined') {
+        return Buffer.from(buffer).toString('base64');
+    } else {
+        throw new Error("Cannot convert ArrayBuffer to Base64 in this environment.");
+    }
+}
+
+function isInlineDataPart(part: Part): part is InlineDataPart {
+    return (
+        typeof part === 'object' &&
+        part !== null &&
+        'inlineData' in part &&
+        typeof (part as any).inlineData === 'object' &&
+        (part as any).inlineData !== null &&
+        typeof (part as any).inlineData.mimeType === 'string' &&
+        typeof (part as any).inlineData.data === 'string'
+    );
+}
+
 interface MultimodalLiveClientEventTypes {
   open: () => void;
-  log: (log: StreamingLog) => void;
+  log: (log: StreamingLog) => void; // این هنوز وجود دارد تا side-panel کار کند
   close: (event: CloseEvent) => void;
   audio: (data: ArrayBuffer) => void;
   content: (data: ServerContent) => void;
@@ -60,34 +85,32 @@ export type MultimodalLiveAPIClientConnection = {
   apiKey?: string;
 };
 
-/**
- * A event-emitting class that manages the connection to the websocket and emits
- * events to the rest of the application.
- * If you dont want to use react you can still use this.
- */
 export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEventTypes> {
   public ws: WebSocket | null = null;
   protected config: LiveConfig | null = null;
   public url: string;
+  private readonly AUDIO_SAMPLE_RATE = 16000;
+  private readonly AUDIO_MIME_TYPE_BASE = `audio/l16`;
+  private readonly AUDIO_MIME_TYPE_WITH_RATE = `${this.AUDIO_MIME_TYPE_BASE};rate=${this.AUDIO_SAMPLE_RATE}`;
+
+  // --- 👇 لاگر داخلی برای ارسال به پنل لاگ، بدون console.log زیاد 👇 ---
+  private logger: ((type: string, message: StreamingLog["message"], count?: number) => void) | null = null;
 
   constructor({ url, apiKey }: MultimodalLiveAPIClientConnection = {}) {
     super();
-    console.log('🔧 Initializing MultimodalLiveClient with URL:', url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`);
+    console.log('🔧 Initializing MultimodalLiveClient...'); // فقط لاگ اولیه
     this.url = url || `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`;
-  }
-
-  log(type: string, message: StreamingLog["message"], count?: number) {
-    const log: StreamingLog = {
-      date: new Date(),
-      type,
-      message,
-      count,
+    // اتصال logger داخلی به emit کردن رویداد log
+    this.logger = (type, message, count) => {
+        const logEntry: StreamingLog = { date: new Date(), type, message, count };
+        this.emit("log", logEntry);
     };
-    this.emit("log", log);
   }
 
+  // تابع log عمومی حذف شد، از logger داخلی استفاده می‌شود
+
   connect(config: LiveConfig): Promise<boolean> {
-    console.log('🔌 Attempting WebSocket connection to:', this.url);
+    this.logger?.("info", "Attempting WebSocket connection...", undefined);
     this.config = config;
 
     if (this.ws) {
@@ -105,16 +128,15 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
             this.receiveParsed(response);
         } catch (e) {
              console.error("Error parsing received binary message:", e);
-             this.log("error", "Error parsing received binary message");
+             this.logger?.("error", "Error parsing received binary message", undefined);
         }
       } else {
-        console.log("Received non-binary message (may be unexpected):", evt.data);
          try {
             const response: LiveIncomingMessage = JSON.parse(evt.data);
             this.receiveParsed(response);
          } catch(e) {
             console.error("Error parsing received text message:", e, evt.data);
-            this.log("error", `Error parsing text message: ${evt.data}`);
+            this.logger?.("error", `Error parsing text message: ${evt.data ? evt.data.substring(0, 100) + '...' : 'empty'}`, undefined);
          }
       }
     });
@@ -123,46 +145,35 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       const onError = (ev: Event) => {
         const message = `Could not connect to "${this.url}"`;
         console.error("WebSocket connection error:", message, ev);
-        this.log(`error.connect`, message);
+        this.logger?.(`error.connect`, message, undefined);
         reject(new Error(message));
       };
       ws.addEventListener("error", onError);
 
       ws.addEventListener("open", (ev: Event) => {
-        console.log('✅ WebSocket connection opened successfully');
-        if (!this.config) {
-          console.error("❌ Config not set when WebSocket opened!");
-          reject("Invalid config state during WebSocket open");
-          return;
-        }
-        this.log(`client.${ev.type}`, `connected to socket`);
+        this.logger?.(`client.${ev.type}`, `connected to socket`, undefined);
         this.emit("open");
 
         this.ws = ws;
 
-        // 1. ارسال پیام Setup اولیه
+        if (!this.config) { // این چک باید بماند
+           console.error("❌ Config not set when WebSocket opened!");
+           this.logger?.("error", "Config not set when WebSocket opened!", undefined);
+           reject("Invalid config state during WebSocket open");
+           return;
+        }
+
         const setupMessage: SetupMessage = {
           setup: this.config,
         };
         this._sendDirect(setupMessage);
-        this.log("client.send.setup", setupMessage);
+        this.logger?.("client.send.setup", setupMessage, undefined); // ارسال به پنل لاگ
 
-        // --- 👇 بخش اضافه شده برای ارسال پیام تحریک کننده 👇 ---
-        // 2. ارسال پیام تحریک کننده برای شروع خوشامدگویی
-        const triggerGreetingMessage: ClientContentMessage = {
-            clientContent: {
-            turns: [{ role: 'user', parts: [{ text: '__START_GREETING__' }] }],
-            turnComplete: true // <<--- به true تغییر یافت
-            }
-        };
-        this._sendDirect(triggerGreetingMessage);
-        this.log("client.send.trigger", triggerGreetingMessage);
-        // --- 👆 پایان بخش اضافه شده 👆 ---
+        // --- پیام‌های تحریک کننده و صدای سکوت حذف شده‌اند ---
 
         ws.removeEventListener("error", onError);
 
         ws.addEventListener("close", (ev: CloseEvent) => {
-          console.log('🔌 WebSocket closed:', ev.code, ev.reason);
           let reason = ev.reason || "";
           if (reason.toLowerCase().includes("error")) {
             const prelude = "ERROR]";
@@ -174,8 +185,7 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
               );
             }
           }
-          console.log('📝 Close reason processed:', reason || 'No reason provided');
-          this.log(
+          this.logger?.(
             `server.close`,
             `disconnected ${reason ? `with reason: ${reason}` : `(code: ${ev.code})`}`,
              ev.code
@@ -189,181 +199,129 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
   }
 
   disconnect(ws?: WebSocket) {
-    console.log('🔌 Attempting to disconnect WebSocket...');
     const wsToClose = ws || this.ws;
-
     if (wsToClose && wsToClose.readyState !== WebSocket.CLOSED && wsToClose.readyState !== WebSocket.CLOSING) {
-      console.log(`🔒 Closing WebSocket connection (readyState: ${wsToClose.readyState})`);
+      this.logger?.("info", `Closing WebSocket connection (readyState: ${wsToClose.readyState})`, undefined);
       wsToClose.close();
       if (this.ws === wsToClose) {
           this.ws = null;
       }
       return true;
-    } else if (wsToClose) {
-         console.log(`⚠️ WebSocket already closing or closed (readyState: ${wsToClose.readyState})`);
-    } else {
-        console.log('⚠️ No active WebSocket reference to disconnect');
-    }
-
-    if (this.ws === wsToClose) {
-        this.ws = null;
     }
-
     return false;
   }
 
-  // متد برای پردازش پیام‌های parse شده
+  private createSilentAudioChunk(durationMs: number): Part {
+      // این تابع دیگر استفاده نمی‌شود اما برای کامل بودن نگه داشته شده
+    const bytesPerSample = 2;
+    const numberOfSamples = Math.floor(this.AUDIO_SAMPLE_RATE * (durationMs / 1000));
+    const bufferSize = numberOfSamples * bytesPerSample;
+    const buffer = new ArrayBuffer(bufferSize);
+    const base64Data = arrayBufferToBase64(buffer);
+    return {
+      inlineData: {
+        mimeType: this.AUDIO_MIME_TYPE_WITH_RATE,
+        data: base64Data,
+      }
+    };
+  }
+
   protected receiveParsed(response: LiveIncomingMessage) {
-     this.log("server.receive", response); // لاگ کردن کل پیام دریافتی
+     // --- 👇 لاگ دریافت پیام حذف شد 👇 ---
+     // this.logger?.("server.receive", response);
 
     if (isToolCallMessage(response)) {
-      this.log("server.toolCall", response);
+      this.logger?.("server.toolCall", response, undefined);
       this.emit("toolcall", response.toolCall);
       return;
     }
     if (isToolCallCancellationMessage(response)) {
-      this.log("server.toolCallCancellation", response);
+      this.logger?.("server.toolCallCancellation", response, undefined);
       this.emit("toolcallcancellation", response.toolCallCancellation);
       return;
     }
-
     if (isSetupCompleteMessage(response)) {
-      this.log("server.setupComplete", response);
+      this.logger?.("server.setupComplete", response, undefined);
       this.emit("setupcomplete");
       return;
     }
-
     if (isServerContentMessage(response)) {
       const { serverContent } = response;
       if (isInterrupted(serverContent)) {
-        this.log("server.interrupted", response);
+        this.logger?.("server.interrupted", response, undefined);
         this.emit("interrupted");
         return;
       }
       if (isTurnComplete(serverContent)) {
-        this.log("server.turnComplete", response);
+        this.logger?.("server.turnComplete", response, undefined);
         this.emit("turncomplete");
       }
-
       if (isModelTurn(serverContent)) {
         let parts: Part[] = serverContent.modelTurn.parts;
-
-        const audioParts = parts.filter(
-          (p) => p.inlineData && p.inlineData.mimeType.startsWith("audio/"),
-        );
-        const base64s = audioParts.map((p) => p.inlineData?.data);
-
+        const audioParts = parts.filter(isInlineDataPart)
+                                .filter(part => part.inlineData.mimeType.startsWith(this.AUDIO_MIME_TYPE_BASE));
         const otherParts = difference(parts, audioParts);
-
         let audioByteLength = 0;
-        base64s.forEach((b64) => {
-          if (b64) {
+        audioParts.forEach((part) => {
+          if (part.inlineData.data) {
             try {
-              const data = base64ToArrayBuffer(b64);
-              this.emit("audio", data);
+              const data = base64ToArrayBuffer(part.inlineData.data);
+              this.emit("audio", data); // رویداد صدا هنوز emit می‌شود
               audioByteLength += data.byteLength;
             } catch (e) {
                console.error("Error decoding base64 audio:", e);
-               this.log("error", "Error decoding base64 audio");
+               this.logger?.("error", "Error decoding base64 audio", undefined);
             }
           }
         });
-        if (audioByteLength > 0) {
-            this.log(`server.audio`, `buffer`, audioByteLength);
-        }
-
-        if (!otherParts.length) {
-          return;
-        }
+        // --- 👇 لاگ بایت صدا حذف شد 👇 ---
+        // if (audioByteLength > 0) {
+        //     this.logger?.(`server.audio`, `buffer`, audioByteLength);
+        // }
+        if (!otherParts.length) return;
 
         parts = otherParts;
         const modelTurnContent: ModelTurn = { modelTurn: { parts } };
         this.emit("content", modelTurnContent);
-        this.log(`server.content`, response);
+        // --- 👇 لاگ محتوای غیرصوتی حذف شد 👇 ---
+        // this.logger?.(`server.content`, response);
       }
     } else {
+      // لاگ پیام ناشناخته را نگه می‌داریم
       console.log("Received unrecognized message structure:", response);
-      this.log("server.unknown", response);
+      this.logger?.("server.unknown", response, undefined);
     }
   }
 
-
-  /**
-   * send realtimeInput, this is base64 chunks of "audio/pcm" and/or "image/jpg"
-   */
   sendRealtimeInput(chunks: GenerativeContentBlob[]) {
-    let hasAudio = false;
-    let hasVideo = false;
-    for (let i = 0; i < chunks.length; i++) {
-      const ch = chunks[i];
-      if (ch.mimeType.includes("audio")) {
-        hasAudio = true;
-      }
-      if (ch.mimeType.includes("image")) {
-        hasVideo = true;
-      }
-      if (hasAudio && hasVideo) {
-        break;
-      }
-    }
-    const message =
-      hasAudio && hasVideo
-        ? "audio + video"
-        : hasAudio
-          ? "audio"
-          : hasVideo
-            ? "video"
-            : "unknown";
-
+    // --- 👇 لاگ ارسال صدا حذف شد 👇 ---
     const data: RealtimeInputMessage = {
-      realtimeInput: {
-        mediaChunks: chunks,
-      },
+      realtimeInput: { mediaChunks: chunks },
     };
     this._sendDirect(data);
-    this.log(`client.realtimeInput.${message}`, data, chunks.length);
+    // this.logger?.(`client.realtimeInput`, `audio/video chunks`, chunks.length);
   }
 
-  /**
-   *  send a response to a function call and provide the id of the functions you are responding to
-   */
   sendToolResponse(toolResponse: ToolResponseMessage["toolResponse"]) {
-    const message: ToolResponseMessage = {
-      toolResponse,
-    };
+    const message: ToolResponseMessage = { toolResponse };
     this._sendDirect(message);
-    this.log(`client.toolResponse`, message);
+    this.logger?.(`client.toolResponse`, message, undefined); // این لاگ معمولا کم تکرار است
   }
 
-  /**
-   * send normal content parts such as { text }
-   */
   send(parts: Part | Part[], turnComplete: boolean = true) {
     parts = Array.isArray(parts) ? parts : [parts];
-    const content: Content = {
-      role: "user",
-      parts,
-    };
-
+    const content: Content = { role: "user", parts };
     const clientContentRequest: ClientContentMessage = {
-      clientContent: {
-        turns: [content],
-        turnComplete,
-      },
+      clientContent: { turns: [content], turnComplete },
     };
-
     this._sendDirect(clientContentRequest);
-    this.log(`client.send.content`, clientContentRequest);
+    this.logger?.(`client.send.content`, clientContentRequest, undefined); // این لاگ هم معمولا کم تکرار است
   }
 
-  /**
-   *  used internally to send all messages
-   *  don't use directly unless trying to send an unsupported message type
-   */
   _sendDirect(request: object) {
     if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
       console.error("WebSocket is not connected or not open. Cannot send message:", request);
-      this.log("error", "Attempted to send message while WebSocket not open");
+      this.logger?.("error", "Attempted to send message while WebSocket not open", undefined);
       return;
     }
     try {
@@ -371,7 +329,7 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
       this.ws.send(str);
     } catch (error) {
         console.error("Error stringifying or sending message:", error, request);
-        this.log("error", `Error sending message: ${error}`);
+        this.logger?.("error", `Error sending message: ${error}`, undefined);
     }
   }
 }