import { Buffer } from 'node:buffer'; import fetch from 'node-fetch'; import express from 'express'; import { speak, languages } from 'google-translate-api-x'; import crypto from 'node:crypto'; import { readSecret, SECRET_KEYS } from './secrets.js'; import { GEMINI_SAFETY } from '../constants.js'; import { getConfigValue, trimTrailingSlash } from '../util.js'; const API_MAKERSUITE = 'https://generativelanguage.googleapis.com'; const API_VERTEX_AI = 'https://us-central1-aiplatform.googleapis.com'; function createWavHeader(dataSize, sampleRate, numChannels = 1, bitsPerSample = 16) { const header = Buffer.alloc(44); header.write('RIFF', 0); header.writeUInt32LE(36 + dataSize, 4); header.write('WAVE', 8); header.write('fmt ', 12); header.writeUInt32LE(16, 16); header.writeUInt16LE(1, 20); header.writeUInt16LE(numChannels, 22); header.writeUInt32LE(sampleRate, 24); header.writeUInt32LE(sampleRate * numChannels * bitsPerSample / 8, 28); header.writeUInt16LE(numChannels * bitsPerSample / 8, 32); header.writeUInt16LE(bitsPerSample, 34); header.write('data', 36); header.writeUInt32LE(dataSize, 40); return header; } function createCompleteWavFile(pcmData, sampleRate) { const header = createWavHeader(pcmData.length, sampleRate); return Buffer.concat([header, pcmData]); } // Vertex AI authentication helper functions export async function getVertexAIAuth(request) { const authMode = request.body.vertexai_auth_mode || 'express'; if (request.body.reverse_proxy) { return { authHeader: `Bearer ${request.body.proxy_password}`, authType: 'proxy', }; } if (authMode === 'express') { const apiKey = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI); if (apiKey) { return { authHeader: `Bearer ${apiKey}`, authType: 'express', }; } throw new Error('API key is required for Vertex AI Express mode'); } else if (authMode === 'full') { // Get service account JSON from backend storage const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT); if (serviceAccountJson) { try { const serviceAccount = JSON.parse(serviceAccountJson); const jwtToken = await generateJWTToken(serviceAccount); const accessToken = await getAccessToken(jwtToken); return { authHeader: `Bearer ${accessToken}`, authType: 'full', }; } catch (error) { console.error('Failed to authenticate with service account:', error); throw new Error(`Service account authentication failed: ${error.message}`); } } throw new Error('Service Account JSON is required for Vertex AI Full mode'); } throw new Error(`Unsupported Vertex AI authentication mode: ${authMode}`); } /** * Generates a JWT token for Google Cloud authentication using service account credentials. * @param {object} serviceAccount Service account JSON object * @returns {Promise} JWT token */ export async function generateJWTToken(serviceAccount) { const now = Math.floor(Date.now() / 1000); const expiry = now + 3600; // 1 hour const header = { alg: 'RS256', typ: 'JWT', }; const payload = { iss: serviceAccount.client_email, scope: 'https://www.googleapis.com/auth/cloud-platform', aud: 'https://oauth2.googleapis.com/token', iat: now, exp: expiry, }; const headerBase64 = Buffer.from(JSON.stringify(header)).toString('base64url'); const payloadBase64 = Buffer.from(JSON.stringify(payload)).toString('base64url'); const signatureInput = `${headerBase64}.${payloadBase64}`; // Create signature using private key const sign = crypto.createSign('RSA-SHA256'); sign.update(signatureInput); const signature = sign.sign(serviceAccount.private_key, 'base64url'); return `${signatureInput}.${signature}`; } export async function getAccessToken(jwtToken) { const response = await fetch('https://oauth2.googleapis.com/token', { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion: jwtToken, }), }); if (!response.ok) { const error = await response.text(); throw new Error(`Failed to get access token: ${error}`); } /** @type {any} */ const data = await response.json(); return data.access_token; } /** * Extracts the project ID from a Service Account JSON object. * @param {object} serviceAccount Service account JSON object * @returns {string} Project ID * @throws {Error} If project ID is not found in the service account */ export function getProjectIdFromServiceAccount(serviceAccount) { if (!serviceAccount || typeof serviceAccount !== 'object') { throw new Error('Invalid service account object'); } const projectId = serviceAccount.project_id; if (!projectId || typeof projectId !== 'string') { throw new Error('Project ID not found in service account JSON'); } return projectId; } /** * Generates Google API URL and headers based on request configuration * @param {express.Request} request Express request object * @param {string} model Model name to use * @param {string} endpoint API endpoint (default: 'generateContent') * @returns {Promise<{url: string, headers: object, apiName: string}>} URL, headers, and API name */ export async function getGoogleApiConfig(request, model, endpoint = 'generateContent') { const useVertexAi = request.body.api === 'vertexai'; const region = request.body.vertexai_region || 'us-central1'; const apiName = useVertexAi ? 'Google Vertex AI' : 'Google AI Studio'; let url; let headers = { 'Content-Type': 'application/json', }; if (useVertexAi) { // Get authentication for Vertex AI const { authHeader, authType } = await getVertexAIAuth(request); if (authType === 'express') { // Express mode: use API key parameter const keyParam = authHeader.replace('Bearer ', ''); const projectId = request.body.vertexai_express_project_id; const baseUrl = region === 'global' ? 'https://aiplatform.googleapis.com' : `https://${region}-aiplatform.googleapis.com`; url = projectId ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}?key=${keyParam}` : `${baseUrl}/v1/publishers/google/models/${model}:${endpoint}?key=${keyParam}`; } else if (authType === 'full') { // Full mode: use project-specific URL with Authorization header // Get project ID from Service Account JSON const serviceAccountJson = readSecret(request.user.directories, SECRET_KEYS.VERTEXAI_SERVICE_ACCOUNT); if (!serviceAccountJson) { throw new Error('Vertex AI Service Account JSON is missing.'); } let projectId; try { const serviceAccount = JSON.parse(serviceAccountJson); projectId = getProjectIdFromServiceAccount(serviceAccount); } catch (error) { throw new Error('Failed to extract project ID from Service Account JSON.'); } // Handle global region differently - no region prefix in hostname url = region === 'global' ? `https://aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}` : `https://${region}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${region}/publishers/google/models/${model}:${endpoint}`; headers['Authorization'] = authHeader; } else { // Proxy mode: use Authorization header const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_VERTEX_AI); url = `${apiUrl}/v1/publishers/google/models/${model}:${endpoint}`; headers['Authorization'] = authHeader; } } else { // Google AI Studio const apiKey = request.body.reverse_proxy ? request.body.proxy_password : readSecret(request.user.directories, SECRET_KEYS.MAKERSUITE); const apiUrl = trimTrailingSlash(request.body.reverse_proxy || API_MAKERSUITE); const apiVersion = getConfigValue('gemini.apiVersion', 'v1beta'); url = `${apiUrl}/${apiVersion}/models/${model}:${endpoint}?key=${apiKey}`; } return { url, headers, apiName }; } export const router = express.Router(); router.post('/caption-image', async (request, response) => { try { const mimeType = request.body.image.split(';')[0].split(':')[1]; const base64Data = request.body.image.split(',')[1]; const model = request.body.model || 'gemini-2.0-flash'; const { url, headers, apiName } = await getGoogleApiConfig(request, model); const body = { contents: [{ role: 'user', parts: [ { text: request.body.prompt }, { inlineData: { mimeType: mimeType, data: base64Data, }, }], }], safetySettings: GEMINI_SAFETY, }; console.debug(`${apiName} captioning request`, model, body); const result = await fetch(url, { body: JSON.stringify(body), method: 'POST', headers: headers, }); if (!result.ok) { const error = await result.json(); console.error(`${apiName} API returned error: ${result.status} ${result.statusText}`, error); return response.status(500).send({ error: true }); } /** @type {any} */ const data = await result.json(); console.info(`${apiName} captioning response`, data); const candidates = data?.candidates; if (!candidates) { return response.status(500).send('No candidates found, image was most likely filtered.'); } const caption = candidates[0].content.parts[0].text; if (!caption) { return response.status(500).send('No caption found'); } return response.json({ caption }); } catch (error) { console.error(error); response.status(500).send('Internal server error'); } }); router.post('/list-voices', (_, response) => { return response.json(languages); }); router.post('/generate-voice', async (request, response) => { try { const text = request.body.text; const voice = request.body.voice ?? 'en'; const result = await speak(text, { to: voice, forceBatch: false }); const buffer = Array.isArray(result) ? Buffer.concat(result.map(x => new Uint8Array(Buffer.from(x.toString(), 'base64')))) : Buffer.from(result.toString(), 'base64'); response.setHeader('Content-Type', 'audio/mpeg'); return response.send(buffer); } catch (error) { console.error('Google Translate TTS generation failed', error); response.status(500).send('Internal server error'); } }); router.post('/list-native-voices', async (_, response) => { try { // Hardcoded Gemini native TTS voices from official documentation // Source: https://ai.google.dev/gemini-api/docs/speech-generation#voices const voices = [ { name: 'Zephyr', voice_id: 'Zephyr', lang: 'en-US', description: 'Bright' }, { name: 'Puck', voice_id: 'Puck', lang: 'en-US', description: 'Upbeat' }, { name: 'Charon', voice_id: 'Charon', lang: 'en-US', description: 'Informative' }, { name: 'Kore', voice_id: 'Kore', lang: 'en-US', description: 'Firm' }, { name: 'Fenrir', voice_id: 'Fenrir', lang: 'en-US', description: 'Excitable' }, { name: 'Leda', voice_id: 'Leda', lang: 'en-US', description: 'Youthful' }, { name: 'Orus', voice_id: 'Orus', lang: 'en-US', description: 'Firm' }, { name: 'Aoede', voice_id: 'Aoede', lang: 'en-US', description: 'Breezy' }, { name: 'Callirhoe', voice_id: 'Callirhoe', lang: 'en-US', description: 'Easy-going' }, { name: 'Autonoe', voice_id: 'Autonoe', lang: 'en-US', description: 'Bright' }, { name: 'Enceladus', voice_id: 'Enceladus', lang: 'en-US', description: 'Breathy' }, { name: 'Iapetus', voice_id: 'Iapetus', lang: 'en-US', description: 'Clear' }, { name: 'Umbriel', voice_id: 'Umbriel', lang: 'en-US', description: 'Easy-going' }, { name: 'Algieba', voice_id: 'Algieba', lang: 'en-US', description: 'Smooth' }, { name: 'Despina', voice_id: 'Despina', lang: 'en-US', description: 'Smooth' }, { name: 'Erinome', voice_id: 'Erinome', lang: 'en-US', description: 'Clear' }, { name: 'Algenib', voice_id: 'Algenib', lang: 'en-US', description: 'Gravelly' }, { name: 'Rasalgethi', voice_id: 'Rasalgethi', lang: 'en-US', description: 'Informative' }, { name: 'Laomedeia', voice_id: 'Laomedeia', lang: 'en-US', description: 'Upbeat' }, { name: 'Achernar', voice_id: 'Achernar', lang: 'en-US', description: 'Soft' }, { name: 'Alnilam', voice_id: 'Alnilam', lang: 'en-US', description: 'Firm' }, { name: 'Schedar', voice_id: 'Schedar', lang: 'en-US', description: 'Even' }, { name: 'Gacrux', voice_id: 'Gacrux', lang: 'en-US', description: 'Mature' }, { name: 'Pulcherrima', voice_id: 'Pulcherrima', lang: 'en-US', description: 'Forward' }, { name: 'Achird', voice_id: 'Achird', lang: 'en-US', description: 'Friendly' }, { name: 'Zubenelgenubi', voice_id: 'Zubenelgenubi', lang: 'en-US', description: 'Casual' }, { name: 'Vindemiatrix', voice_id: 'Vindemiatrix', lang: 'en-US', description: 'Gentle' }, { name: 'Sadachbia', voice_id: 'Sadachbia', lang: 'en-US', description: 'Lively' }, { name: 'Sadaltager', voice_id: 'Sadaltager', lang: 'en-US', description: 'Knowledgeable' }, { name: 'Sulafat', voice_id: 'Sulafat', lang: 'en-US', description: 'Warm' }, ]; return response.json({ voices }); } catch (error) { console.error('Failed to return Google TTS voices:', error); response.sendStatus(500); } }); router.post('/generate-native-tts', async (request, response) => { try { const { text, voice, model } = request.body; const { url, headers, apiName } = await getGoogleApiConfig(request, model); console.debug(`${apiName} TTS request`, { model, text, voice }); const requestBody = { contents: [{ role: 'user', parts: [{ text: text }], }], generationConfig: { responseModalities: ['AUDIO'], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice, }, }, }, }, safetySettings: GEMINI_SAFETY, }; const result = await fetch(url, { method: 'POST', headers: headers, body: JSON.stringify(requestBody), }); if (!result.ok) { const errorText = await result.text(); console.error(`${apiName} TTS API error: ${result.status} ${result.statusText}`, errorText); const errorMessage = JSON.parse(errorText).error?.message || 'TTS generation failed.'; return response.status(result.status).json({ error: errorMessage }); } /** @type {any} */ const data = await result.json(); const audioPart = data?.candidates?.[0]?.content?.parts?.[0]; const audioData = audioPart?.inlineData?.data; const mimeType = audioPart?.inlineData?.mimeType; if (!audioData) { return response.status(500).json({ error: 'No audio data found in response' }); } const audioBuffer = Buffer.from(audioData, 'base64'); //If the audio is raw PCM, wrap it in a WAV header and send it. if (mimeType && mimeType.toLowerCase().includes('audio/l16')) { const rateMatch = mimeType.match(/rate=(\d+)/); const sampleRate = rateMatch ? parseInt(rateMatch[1], 10) : 24000; const pcmData = audioBuffer; // Create a complete, playable WAV file buffer. const wavBuffer = createCompleteWavFile(pcmData, sampleRate); // Send the WAV file directly to the browser. This is much faster. response.setHeader('Content-Type', 'audio/wav'); return response.send(wavBuffer); } // Fallback for any other audio format Google might send in the future. response.setHeader('Content-Type', mimeType || 'application/octet-stream'); response.send(audioBuffer); } catch (error) { console.error('Google TTS generation failed:', error); if (!response.headersSent) { return response.status(500).json({ error: 'Internal server error during TTS generation' }); } return response.end(); } }); router.post('/generate-image', async (request, response) => { try { const model = request.body.model || 'imagen-3.0-generate-002'; const { url, headers, apiName } = await getGoogleApiConfig(request, model, 'predict'); // AI Studio is stricter than Vertex AI. const isVertex = request.body.api === 'vertexai'; // Is it even worth it? const isDeprecated = model.startsWith('imagegeneration'); const requestBody = { instances: [{ prompt: request.body.prompt || '', }], parameters: { sampleCount: 1, seed: isVertex ? Number(request.body.seed ?? Math.floor(Math.random() * 1000000)) : undefined, enhancePrompt: isVertex ? Boolean(request.body.enhance ?? false) : undefined, negativePrompt: isVertex ? (request.body.negative_prompt || undefined) : undefined, aspectRatio: String(request.body.aspect_ratio || '1:1'), personGeneration: !isDeprecated ? 'allow_all' : undefined, language: isVertex ? 'auto' : undefined, safetySetting: !isDeprecated ? (isVertex ? 'block_only_high' : 'block_low_and_above') : undefined, addWatermark: isVertex ? false : undefined, outputOptions: { mimeType: 'image/jpeg', compressionQuality: 100, }, }, }; console.debug(`${apiName} image generation request:`, model, requestBody); const result = await fetch(url, { method: 'POST', headers: headers, body: JSON.stringify(requestBody), }); if (!result.ok) { const errorText = await result.text(); console.warn(`${apiName} image generation error: ${result.status} ${result.statusText}`, errorText); return response.sendStatus(500); } /** @type {any} */ const data = await result.json(); const imagePart = data?.predictions?.[0]?.bytesBase64Encoded; if (!imagePart) { console.warn(`${apiName} image generation error: No image data found in response`); return response.sendStatus(500); } return response.send({ image: imagePart }); } catch (error) { console.error('Google Image generation failed:', error); if (!response.headersSent) { return response.sendStatus(500); } return response.end(); } });