Files
Meeting_Assistant/client/src/main.js
egg 49dba2c43e fix: Improve microphone permission handling and audio capture robustness
- Add device enumeration check before attempting to capture audio
- Use simpler audio constraints (audio: true) instead of specific options
- Add fallback to explicit device ID if simple constraints fail
- Add more descriptive error messages for different failure modes
- Enhance Electron permission handlers with better logging
- Add setDevicePermissionHandler for audio device access
- Include 'microphone' in allowed permissions list

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-17 16:47:09 +08:00

695 lines
21 KiB
JavaScript

const { app, BrowserWindow, ipcMain, session } = require("electron");
const path = require("path");
const fs = require("fs");
const { spawn } = require("child_process");
const os = require("os");
let mainWindow;
let sidecarProcess;
let sidecarReady = false;
let streamingActive = false;
let appConfig = null;
let activeWhisperConfig = null;
// Backend sidecar state
let backendProcess = null;
let backendReady = false;
/**
* Load configuration from external config.json
* Config file location:
* - Development: client/config.json
* - Packaged: <app>/resources/config.json
*/
function loadConfig() {
console.log("=== Config Loading Debug ===");
console.log("app.isPackaged:", app.isPackaged);
console.log("process.resourcesPath:", process.resourcesPath);
console.log("__dirname:", __dirname);
const configPaths = [
// Packaged app: resources folder
app.isPackaged ? path.join(process.resourcesPath, "config.json") : null,
// Development: client folder
path.join(__dirname, "..", "config.json"),
// Fallback: same directory
path.join(__dirname, "config.json"),
].filter(Boolean);
console.log("Config search paths:", configPaths);
for (const configPath of configPaths) {
const exists = fs.existsSync(configPath);
console.log(`Checking: ${configPath} - exists: ${exists}`);
try {
if (exists) {
// Use utf-8 and strip BOM if present (Windows Notepad adds BOM)
let configData = fs.readFileSync(configPath, "utf-8");
// Remove UTF-8 BOM if present
if (configData.charCodeAt(0) === 0xFEFF) {
configData = configData.slice(1);
}
appConfig = JSON.parse(configData);
console.log("Config loaded from:", configPath);
console.log("Config content:", JSON.stringify(appConfig, null, 2));
console.log("Whisper config:", appConfig.whisper);
return appConfig;
}
} catch (error) {
console.warn("Failed to load config from:", configPath, error.message);
}
}
// Default configuration
appConfig = {
apiBaseUrl: "http://localhost:8000/api",
uploadTimeout: 600000,
appTitle: "Meeting Assistant",
whisper: {
model: "medium",
device: "cpu",
compute: "int8"
}
};
console.log("WARNING: No config.json found, using defaults");
console.log("Default config:", JSON.stringify(appConfig, null, 2));
return appConfig;
}
/**
* Get config.json path for backend sidecar
*/
function getConfigPath() {
if (app.isPackaged) {
return path.join(process.resourcesPath, "config.json");
}
return path.join(__dirname, "..", "config.json");
}
/**
* Start backend sidecar process (FastAPI server)
* Only starts if backend.embedded is true in config
*/
function startBackendSidecar() {
const backendConfig = appConfig?.backend || {};
// Check if embedded backend is enabled
if (!backendConfig.embedded) {
console.log("Backend embedded mode disabled, using remote backend at:", appConfig?.apiBaseUrl);
backendReady = true; // Assume remote backend is ready
return;
}
console.log("Starting embedded backend sidecar...");
const backendDir = app.isPackaged
? path.join(process.resourcesPath, "backend")
: path.join(__dirname, "..", "..", "backend");
// Determine the backend executable path
let backendExecutable;
let backendArgs = [];
if (app.isPackaged) {
// Packaged app: use PyInstaller-built executable
if (process.platform === "win32") {
backendExecutable = path.join(backendDir, "backend", "backend.exe");
} else {
backendExecutable = path.join(backendDir, "backend", "backend");
}
// Pass config path
backendArgs = ["--config", getConfigPath()];
} else {
// Development mode: use Python script with venv
const backendScript = path.join(backendDir, "run_server.py");
if (!fs.existsSync(backendScript)) {
console.log("Backend script not found at:", backendScript);
console.log("Backend sidecar will not be available.");
return;
}
// Check for virtual environment Python
let venvPython;
if (process.platform === "win32") {
venvPython = path.join(backendDir, "venv", "Scripts", "python.exe");
} else {
venvPython = path.join(backendDir, "venv", "bin", "python");
}
backendExecutable = fs.existsSync(venvPython) ? venvPython : "python3";
backendArgs = [backendScript, "--config", getConfigPath()];
}
if (!fs.existsSync(backendExecutable) && app.isPackaged) {
console.log("Backend executable not found at:", backendExecutable);
console.log("Backend sidecar will not be available.");
return;
}
try {
console.log("Starting backend with:", backendExecutable, backendArgs.join(" "));
backendProcess = spawn(backendExecutable, backendArgs, {
cwd: backendDir,
stdio: ["pipe", "pipe", "pipe"],
env: process.env,
});
backendProcess.stdout.on("data", (data) => {
console.log("Backend:", data.toString().trim());
});
backendProcess.stderr.on("data", (data) => {
console.log("Backend:", data.toString().trim());
});
backendProcess.on("close", (code) => {
console.log(`Backend exited with code ${code}`);
backendReady = false;
backendProcess = null;
});
backendProcess.on("error", (err) => {
console.error("Backend error:", err.message);
backendReady = false;
});
} catch (error) {
console.error("Failed to start backend:", error);
}
}
/**
* Wait for backend to be ready by polling health endpoint
* @param {number} maxAttempts - Maximum number of attempts (default 30)
* @param {number} intervalMs - Interval between attempts in ms (default 1000)
* @returns {Promise<boolean>} - True if backend is ready, false if timeout
*/
async function waitForBackendReady(maxAttempts = 30, intervalMs = 1000) {
const backendConfig = appConfig?.backend || {};
// If embedded mode is disabled, assume backend is ready
if (!backendConfig.embedded) {
backendReady = true;
return true;
}
const host = backendConfig.host || "127.0.0.1";
const port = backendConfig.port || 8000;
const healthUrl = `http://${host}:${port}/api/health`;
console.log(`Waiting for backend at ${healthUrl}...`);
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
const http = require("http");
const ready = await new Promise((resolve) => {
const req = http.get(healthUrl, (res) => {
resolve(res.statusCode === 200);
});
req.on("error", () => resolve(false));
req.setTimeout(2000, () => {
req.destroy();
resolve(false);
});
});
if (ready) {
console.log(`Backend ready after ${attempt} attempt(s)`);
backendReady = true;
return true;
}
} catch (e) {
// Ignore errors, will retry
}
console.log(`Backend health check attempt ${attempt}/${maxAttempts} failed, retrying...`);
await new Promise((resolve) => setTimeout(resolve, intervalMs));
}
console.error(`Backend did not become ready after ${maxAttempts} attempts`);
return false;
}
/**
* Stop backend sidecar process
*/
function stopBackendSidecar() {
if (!backendProcess) {
return;
}
console.log("Stopping backend sidecar...");
// Send SIGTERM first
backendProcess.kill("SIGTERM");
// Force kill after 5 seconds if still running
const forceKillTimeout = setTimeout(() => {
if (backendProcess) {
console.log("Force killing backend sidecar...");
backendProcess.kill("SIGKILL");
}
}, 5000);
backendProcess.on("close", () => {
clearTimeout(forceKillTimeout);
backendProcess = null;
backendReady = false;
});
}
function createWindow() {
// Set window title from config
const windowTitle = appConfig?.appTitle || "Meeting Assistant";
mainWindow = new BrowserWindow({
width: 1200,
height: 800,
title: windowTitle,
webPreferences: {
nodeIntegration: false,
contextIsolation: true,
preload: path.join(__dirname, "preload.js"),
},
});
mainWindow.loadFile(path.join(__dirname, "pages", "login.html"));
mainWindow.on("closed", () => {
mainWindow = null;
});
}
function startSidecar() {
const sidecarDir = app.isPackaged
? path.join(process.resourcesPath, "sidecar")
: path.join(__dirname, "..", "..", "sidecar");
// Determine the sidecar executable path based on packaging and platform
let sidecarExecutable;
let sidecarArgs = [];
if (app.isPackaged) {
// Packaged app: use PyInstaller-built executable
if (process.platform === "win32") {
sidecarExecutable = path.join(sidecarDir, "transcriber", "transcriber.exe");
} else if (process.platform === "darwin") {
sidecarExecutable = path.join(sidecarDir, "transcriber", "transcriber");
} else {
sidecarExecutable = path.join(sidecarDir, "transcriber", "transcriber");
}
} else {
// Development mode: use Python script with venv
const sidecarScript = path.join(sidecarDir, "transcriber.py");
if (!fs.existsSync(sidecarScript)) {
console.log("Sidecar script not found at:", sidecarScript);
console.log("Transcription will not be available.");
return;
}
// Check for virtual environment Python
let venvPython;
if (process.platform === "win32") {
venvPython = path.join(sidecarDir, "venv", "Scripts", "python.exe");
} else {
venvPython = path.join(sidecarDir, "venv", "bin", "python");
}
sidecarExecutable = fs.existsSync(venvPython) ? venvPython : "python3";
sidecarArgs = [sidecarScript];
}
if (!fs.existsSync(sidecarExecutable)) {
console.log("Sidecar executable not found at:", sidecarExecutable);
console.log("Transcription will not be available.");
return;
}
try {
// Get Whisper configuration from config.json or environment variables
console.log("=== Whisper Config Resolution ===");
console.log("appConfig:", appConfig);
console.log("appConfig?.whisper:", appConfig?.whisper);
const whisperConfig = appConfig?.whisper || {};
console.log("whisperConfig (resolved):", whisperConfig);
console.log("process.env.WHISPER_MODEL:", process.env.WHISPER_MODEL);
console.log("whisperConfig.model:", whisperConfig.model);
const whisperEnv = {
...process.env,
WHISPER_MODEL: process.env.WHISPER_MODEL || whisperConfig.model || "medium",
WHISPER_DEVICE: process.env.WHISPER_DEVICE || whisperConfig.device || "cpu",
WHISPER_COMPUTE: process.env.WHISPER_COMPUTE || whisperConfig.compute || "int8",
};
// Store the active whisper config for status reporting
activeWhisperConfig = {
model: whisperEnv.WHISPER_MODEL,
device: whisperEnv.WHISPER_DEVICE,
compute: whisperEnv.WHISPER_COMPUTE,
configSource: appConfig?.whisper ? "config.json" : "defaults"
};
console.log("=== Final Whisper Environment ===");
console.log("WHISPER_MODEL:", whisperEnv.WHISPER_MODEL);
console.log("WHISPER_DEVICE:", whisperEnv.WHISPER_DEVICE);
console.log("WHISPER_COMPUTE:", whisperEnv.WHISPER_COMPUTE);
console.log("Starting sidecar with:", sidecarExecutable, sidecarArgs.join(" "));
console.log("Active Whisper config:", activeWhisperConfig);
sidecarProcess = spawn(sidecarExecutable, sidecarArgs, {
cwd: sidecarDir,
stdio: ["pipe", "pipe", "pipe"],
env: whisperEnv,
});
// Handle stdout (JSON responses)
sidecarProcess.stdout.on("data", (data) => {
const lines = data.toString().split("\n").filter(l => l.trim());
for (const line of lines) {
try {
const msg = JSON.parse(line);
console.log("Sidecar response:", msg);
if (msg.status === "ready") {
sidecarReady = true;
console.log("Sidecar is ready");
}
// Forward streaming segment to renderer
if (msg.segment_id !== undefined && mainWindow) {
mainWindow.webContents.send("transcription-segment", msg);
}
// Forward stream status changes
if (msg.status === "streaming" && mainWindow) {
mainWindow.webContents.send("stream-started", msg);
}
if (msg.status === "stream_stopped" && mainWindow) {
mainWindow.webContents.send("stream-stopped", msg);
}
// Legacy: file-based transcription result
if (msg.result !== undefined && mainWindow) {
mainWindow.webContents.send("transcription-result", msg.result);
}
// Forward model download progress to renderer
if (msg.status === "downloading_model" && mainWindow) {
mainWindow.webContents.send("model-download-progress", msg);
}
// Forward model downloaded status
if (msg.status === "model_downloaded" && mainWindow) {
mainWindow.webContents.send("model-download-progress", msg);
}
// Forward model loading status
if (msg.status === "loading_model" && mainWindow) {
mainWindow.webContents.send("model-download-progress", msg);
}
// Forward model loaded status
if (msg.status === "model_loaded" && mainWindow) {
mainWindow.webContents.send("model-download-progress", msg);
}
} catch (e) {
console.log("Sidecar output:", line);
}
}
});
sidecarProcess.stderr.on("data", (data) => {
console.log("Sidecar:", data.toString().trim());
});
sidecarProcess.on("close", (code) => {
console.log(`Sidecar exited with code ${code}`);
sidecarReady = false;
streamingActive = false;
});
sidecarProcess.on("error", (err) => {
console.error("Sidecar error:", err.message);
});
} catch (error) {
console.error("Failed to start sidecar:", error);
}
}
app.whenReady().then(async () => {
// Load configuration first
loadConfig();
// Grant microphone permission automatically
session.defaultSession.setPermissionRequestHandler((webContents, permission, callback, details) => {
console.log(`Permission request: ${permission}`, details);
// Allow all media-related permissions
const allowedPermissions = ['media', 'mediaKeySystem', 'audioCapture', 'microphone'];
if (allowedPermissions.includes(permission)) {
console.log(`Granting permission: ${permission}`);
callback(true);
} else {
console.log(`Denying permission: ${permission}`);
callback(false);
}
});
// Also handle permission check (for some Electron versions)
session.defaultSession.setPermissionCheckHandler((webContents, permission, requestingOrigin, details) => {
console.log(`Permission check: ${permission}`, { requestingOrigin, details });
const allowedPermissions = ['media', 'mediaKeySystem', 'audioCapture', 'microphone'];
return allowedPermissions.includes(permission);
});
// Set device permission handler for media devices
session.defaultSession.setDevicePermissionHandler((details) => {
console.log('Device permission request:', details);
// Allow all audio devices
if (details.deviceType === 'audio' || details.deviceType === 'hid') {
return true;
}
return false;
});
// Start backend sidecar if embedded mode is enabled
startBackendSidecar();
// Wait for backend to be ready before creating window
const backendConfig = appConfig?.backend || {};
if (backendConfig.embedded) {
const ready = await waitForBackendReady();
if (!ready) {
const { dialog } = require("electron");
dialog.showErrorBox(
"Backend Startup Failed",
"The backend server failed to start within 30 seconds. Please check the logs for details."
);
app.quit();
return;
}
}
createWindow();
startSidecar();
app.on("activate", () => {
if (BrowserWindow.getAllWindows().length === 0) {
createWindow();
}
});
});
app.on("window-all-closed", () => {
// Stop transcriber sidecar
if (sidecarProcess) {
try {
sidecarProcess.stdin.write(JSON.stringify({ action: "quit" }) + "\n");
} catch (e) {}
sidecarProcess.kill();
}
// Stop backend sidecar
stopBackendSidecar();
if (process.platform !== "darwin") {
app.quit();
}
});
// IPC handlers
// Get app configuration (for renderer to initialize API settings)
ipcMain.handle("get-config", () => {
return appConfig;
});
ipcMain.handle("navigate", (event, page) => {
mainWindow.loadFile(path.join(__dirname, "pages", `${page}.html`));
});
ipcMain.handle("get-sidecar-status", () => {
return {
ready: sidecarReady,
streaming: streamingActive,
whisper: activeWhisperConfig
};
});
// Get backend status (for renderer to check backend readiness)
ipcMain.handle("get-backend-status", () => {
const backendConfig = appConfig?.backend || {};
const host = backendConfig.host || "127.0.0.1";
const port = backendConfig.port || 8000;
return {
ready: backendReady,
embedded: backendConfig.embedded || false,
url: `http://${host}:${port}`
};
});
// === Streaming Mode IPC Handlers ===
ipcMain.handle("start-recording-stream", async () => {
if (!sidecarProcess || !sidecarReady) {
return { error: "Sidecar not ready" };
}
if (streamingActive) {
return { error: "Stream already active" };
}
return new Promise((resolve) => {
const responseHandler = (data) => {
const lines = data.toString().split("\n").filter(l => l.trim());
for (const line of lines) {
try {
const msg = JSON.parse(line);
if (msg.status === "streaming" || msg.error) {
sidecarProcess.stdout.removeListener("data", responseHandler);
if (msg.status === "streaming") {
streamingActive = true;
}
resolve(msg);
return;
}
} catch (e) {}
}
};
sidecarProcess.stdout.on("data", responseHandler);
sidecarProcess.stdin.write(JSON.stringify({ action: "start_stream" }) + "\n");
setTimeout(() => {
sidecarProcess.stdout.removeListener("data", responseHandler);
resolve({ error: "Start stream timeout" });
}, 5000);
});
});
ipcMain.handle("stream-audio-chunk", async (event, base64Audio) => {
if (!sidecarProcess || !sidecarReady || !streamingActive) {
return { error: "Stream not active" };
}
try {
const cmd = JSON.stringify({ action: "audio_chunk", data: base64Audio }) + "\n";
sidecarProcess.stdin.write(cmd);
return { sent: true };
} catch (e) {
return { error: e.message };
}
});
ipcMain.handle("stop-recording-stream", async () => {
if (!sidecarProcess || !streamingActive) {
return { error: "No active stream" };
}
return new Promise((resolve) => {
const responseHandler = (data) => {
const lines = data.toString().split("\n").filter(l => l.trim());
for (const line of lines) {
try {
const msg = JSON.parse(line);
if (msg.status === "stream_stopped" || msg.error) {
sidecarProcess.stdout.removeListener("data", responseHandler);
streamingActive = false;
resolve(msg);
return;
}
} catch (e) {}
}
};
sidecarProcess.stdout.on("data", responseHandler);
sidecarProcess.stdin.write(JSON.stringify({ action: "stop_stream" }) + "\n");
setTimeout(() => {
sidecarProcess.stdout.removeListener("data", responseHandler);
streamingActive = false;
resolve({ error: "Stop stream timeout" });
}, 10000);
});
});
// === Legacy File-based Handlers (kept for fallback) ===
ipcMain.handle("save-audio-file", async (event, arrayBuffer) => {
const tempDir = os.tmpdir();
const tempFile = path.join(tempDir, `recording_${Date.now()}.webm`);
const buffer = Buffer.from(arrayBuffer);
fs.writeFileSync(tempFile, buffer);
return tempFile;
});
ipcMain.handle("transcribe-audio", async (event, audioFilePath) => {
if (!sidecarProcess || !sidecarReady) {
return { error: "Sidecar not ready" };
}
return new Promise((resolve) => {
const responseHandler = (data) => {
const lines = data.toString().split("\n").filter(l => l.trim());
for (const line of lines) {
try {
const msg = JSON.parse(line);
if (msg.result !== undefined || msg.error) {
sidecarProcess.stdout.removeListener("data", responseHandler);
// Delete temp file after transcription
try {
if (fs.existsSync(audioFilePath)) {
fs.unlinkSync(audioFilePath);
}
} catch (e) {
console.error("Failed to delete temp file:", e);
}
resolve(msg);
return;
}
} catch (e) {}
}
};
sidecarProcess.stdout.on("data", responseHandler);
const cmd = JSON.stringify({ action: "transcribe", file: audioFilePath }) + "\n";
sidecarProcess.stdin.write(cmd);
setTimeout(() => {
sidecarProcess.stdout.removeListener("data", responseHandler);
// Delete temp file on timeout too
try {
if (fs.existsSync(audioFilePath)) {
fs.unlinkSync(audioFilePath);
}
} catch (e) {}
resolve({ error: "Transcription timeout" });
}, 60000);
});
});