diff --git a/.env.example b/.env.example index 445933a67..900deab26 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,8 @@ # Use the LexVoice repository `.env` as the single source of truth; its `run.sh` # injects LiveKit, room-input, input-source, role-device, agent, media, and debug # settings into the frontend process when it starts `make start_ui`. +# Do not create `agent-starter-react/.env` for an integrated run; Next.js would +# load it as a second configuration source. # # Only create `agent-starter-react/.env.local` for standalone frontend # development where this repository is launched directly with `pnpm dev`. @@ -11,7 +13,7 @@ # For sandbox gateway deployment, see the LexVoice repository's # `deploy/liveavatar_gateway/.env.example.gateway` and -# `deploy/liveavatar_gateway/.env.example.sandbox` reference files. +# `deploy/liveavatar_gateway/.env.injection.example` reference files. # # `OBSERVABILITY_ENABLED=1` uses the same unified switch as the backend. When # enabled by the LexVoice runtime, browser-side probes publish LiveKit data diff --git a/app/api/session/stop/route.ts b/app/api/session/stop/route.ts index add849012..05165c6ff 100644 --- a/app/api/session/stop/route.ts +++ b/app/api/session/stop/route.ts @@ -9,6 +9,7 @@ import { isValidConnectionRoomId, } from '@/lib/connection-room-id'; import { + isLiveKitRoomNotFoundError, resolveRoomInputStopUrls as resolveConfiguredRoomInputStopUrls, resolveLiveKitHttpUrl, } from '@/lib/session-stop'; @@ -258,6 +259,10 @@ async function deleteLiveKitRoom(roomName: string): Promise { await roomService.deleteRoom(roomName); return { target: 'livekit_room', ok: true }; } catch (error) { + if (isLiveKitRoomNotFoundError(error)) { + return { target: 'livekit_room', ok: true, skipped: true, status: 404 }; + } + return { target: 'livekit_room', ok: false, diff --git a/components/livekit/filtered-audio-renderer.tsx b/components/livekit/filtered-audio-renderer.tsx index 704a29aa0..5610d99e3 100644 --- a/components/livekit/filtered-audio-renderer.tsx +++ b/components/livekit/filtered-audio-renderer.tsx @@ -10,6 +10,7 @@ import { Track, } from 'livekit-client'; import { useRemoteParticipants, useRoomContext } from '@livekit/components-react'; +import { buildBrowserAudioPlaybackDiagnostics } from '@/lib/browser-audio-capture'; import { startMediaTrackAudioObserver } from '@/lib/frontend-audio-observer'; import { FRONTEND_EVENTS, @@ -399,6 +400,15 @@ export function FilteredAudioRenderer({ if (!playbackSource || playbackObserverStops.has(elementKey)) { return; } + const playbackDiagnostics = buildBrowserAudioPlaybackDiagnostics( + participantIdentity, + trackName, + audioElements.values(), + createdAudioElement + ); + const logPlaybackDiagnostics = + playbackDiagnostics.activeAudioElementCount === 1 ? console.info : console.warn; + logPlaybackDiagnostics('[browser-audio] playback diagnostics', playbackDiagnostics); pendingPlayback.delete(elementKey); startPlaybackObserver( elementKey, diff --git a/hooks/useBrowserSourceClient.ts b/hooks/useBrowserSourceClient.ts index 778715168..b85aa42e3 100644 --- a/hooks/useBrowserSourceClient.ts +++ b/hooks/useBrowserSourceClient.ts @@ -11,6 +11,11 @@ import { createLocalVideoTrack, } from 'livekit-client'; import type { AppConfig } from '@/app-config'; +import { + BROWSER_AUDIO_CONSTRAINTS, + assertBrowserEchoCancellationActive, + inspectBrowserAudioCapture, +} from '@/lib/browser-audio-capture'; import { startMediaTrackVadObserver } from '@/lib/frontend-vad-observer'; import { FRONTEND_EVENTS, @@ -23,12 +28,6 @@ const BROWSER_VIDEO_TRACK_NAME = 'browser_video_track'; const DEFAULT_BROWSER_MEDIA_STREAM_NAME = 'browser_input'; const BROWSER_VIDEO_DEFAULT_ENABLED = true; const BROWSER_VIDEO_STATS_INTERVAL_MS = 5000; -const BROWSER_AUDIO_CONSTRAINTS: MediaTrackConstraints = { - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, -}; - interface BrowserSourceRuntime { audioTrack: LocalAudioTrack | null; videoTrack: LocalVideoTrack | null; @@ -135,6 +134,7 @@ export function useBrowserSourceClient( audioTrack.mediaStreamTrack.enabled = runtime.audioEnabled; try { + logBrowserAudioCaptureDiagnostics(captureTrack); const publication = await room.localParticipant.publishTrack(audioTrack, { name: BROWSER_AUDIO_TRACK_NAME, source: Track.Source.Microphone, @@ -541,6 +541,15 @@ function buildAudioCaptureOptions(deviceId: string | null) { }; } +function logBrowserAudioCaptureDiagnostics(track: MediaStreamTrack) { + const diagnostics = inspectBrowserAudioCapture( + track, + navigator.mediaDevices.getSupportedConstraints() + ); + console.info('[browser-audio] capture diagnostics', diagnostics); + assertBrowserEchoCancellationActive(diagnostics); +} + function syncTrackEnabled(track: LocalAudioTrack | LocalVideoTrack | null, enabled: boolean) { if (!track) return; diff --git a/lib/browser-audio-capture.ts b/lib/browser-audio-capture.ts new file mode 100644 index 000000000..4317d181f --- /dev/null +++ b/lib/browser-audio-capture.ts @@ -0,0 +1,66 @@ +export const BROWSER_AUDIO_CONSTRAINTS: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, +}; + +type InspectableAudioTrack = Pick; + +export interface BrowserAudioCaptureDiagnostics { + trackId: string; + supported: MediaTrackSupportedConstraints; + constraints: MediaTrackConstraints; + settings: MediaTrackSettings; +} + +export interface BrowserAudioPlaybackDiagnostics { + participantIdentity: string; + trackName: string; + activeAudioElementCount: number; + paused: boolean; + readyState: number; +} + +type BrowserAudioElementState = Pick; + +export function buildBrowserAudioPlaybackDiagnostics( + participantIdentity: string, + trackName: string, + audioElements: Iterable, + currentElement: BrowserAudioElementState +): BrowserAudioPlaybackDiagnostics { + return { + participantIdentity, + trackName, + activeAudioElementCount: Array.from(audioElements).filter( + (element) => !element.paused && !element.ended && element.readyState >= 2 + ).length, + paused: currentElement.paused, + readyState: currentElement.readyState, + }; +} + +export function inspectBrowserAudioCapture( + track: InspectableAudioTrack, + supported: MediaTrackSupportedConstraints +): BrowserAudioCaptureDiagnostics { + const constraints = track.getConstraints(); + const settings = track.getSettings(); + + return { + trackId: track.id, + supported, + constraints, + settings, + }; +} + +export function assertBrowserEchoCancellationActive( + diagnostics: BrowserAudioCaptureDiagnostics +): void { + if (diagnostics.supported.echoCancellation && diagnostics.settings.echoCancellation !== true) { + throw new Error( + 'Browser echo cancellation was requested but is not active on the microphone track.' + ); + } +} diff --git a/lib/session-stop.ts b/lib/session-stop.ts index ee3a23b6d..8c81dfcba 100644 --- a/lib/session-stop.ts +++ b/lib/session-stop.ts @@ -33,6 +33,15 @@ export function resolveLiveKitHttpUrl(liveKitUrl?: string | null): string | unde return normalized; } +export function isLiveKitRoomNotFoundError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; + } + + const { status, code } = error as { status?: unknown; code?: unknown }; + return status === 404 && code === 'not_found'; +} + function addRoomInputStopUrl(urls: Set, rawUrl?: string | null): void { const stopUrl = normalizeRoomInputControlUrl(rawUrl || '', 'stop'); if (stopUrl) { diff --git a/tests/browser-audio-capture.test.mjs b/tests/browser-audio-capture.test.mjs new file mode 100644 index 000000000..292fe1b63 --- /dev/null +++ b/tests/browser-audio-capture.test.mjs @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +const { + BROWSER_AUDIO_CONSTRAINTS, + assertBrowserEchoCancellationActive, + buildBrowserAudioPlaybackDiagnostics, + inspectBrowserAudioCapture, +} = await import('../lib/browser-audio-capture.ts'); + +test('browser audio capture requests WebRTC audio processing', () => { + assert.deepEqual(BROWSER_AUDIO_CONSTRAINTS, { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); +}); + +test('browser audio capture reports requested and effective settings', () => { + const diagnostics = inspectBrowserAudioCapture( + { + id: 'audio-track-1', + getConstraints: () => BROWSER_AUDIO_CONSTRAINTS, + getSettings: () => ({ + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + channelCount: 1, + sampleRate: 48000, + }), + }, + { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + } + ); + + assert.equal(diagnostics.trackId, 'audio-track-1'); + assert.equal(diagnostics.settings.echoCancellation, true); + assert.equal(diagnostics.settings.noiseSuppression, true); + assert.deepEqual(diagnostics.constraints, BROWSER_AUDIO_CONSTRAINTS); +}); + +test('browser audio capture fails when supported AEC is not effective', () => { + const diagnostics = inspectBrowserAudioCapture( + { + id: 'audio-track-2', + getConstraints: () => BROWSER_AUDIO_CONSTRAINTS, + getSettings: () => ({ echoCancellation: false }), + }, + { echoCancellation: true } + ); + + assert.equal(diagnostics.settings.echoCancellation, false); + assert.throws( + () => assertBrowserEchoCancellationActive(diagnostics), + /echo cancellation was requested but is not active/i + ); +}); + +test('browser audio capture does not claim unsupported AEC is active', () => { + const diagnostics = inspectBrowserAudioCapture( + { + id: 'audio-track-3', + getConstraints: () => BROWSER_AUDIO_CONSTRAINTS, + getSettings: () => ({}), + }, + { echoCancellation: false } + ); + + assert.equal(diagnostics.supported.echoCancellation, false); + assert.equal(diagnostics.settings.echoCancellation, undefined); + assert.doesNotThrow(() => assertBrowserEchoCancellationActive(diagnostics)); +}); + +test('browser playback diagnostics identify the active output and element count', () => { + const playingElement = { paused: false, ended: false, readyState: 4 }; + const pausedElement = { paused: true, ended: false, readyState: 4 }; + + assert.deepEqual( + buildBrowserAudioPlaybackDiagnostics( + 'agent-AJ_123', + 'roomio_audio', + [playingElement, pausedElement], + playingElement + ), + { + participantIdentity: 'agent-AJ_123', + trackName: 'roomio_audio', + activeAudioElementCount: 1, + paused: false, + readyState: 4, + } + ); +}); diff --git a/tests/session-stop.test.mjs b/tests/session-stop.test.mjs index ec93b453b..dac0924c6 100644 --- a/tests/session-stop.test.mjs +++ b/tests/session-stop.test.mjs @@ -2,7 +2,11 @@ import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; import { test } from 'node:test'; import { readAgentWorkerStateFromLog } from '../lib/agent-worker-readiness.ts'; -import { resolveLiveKitHttpUrl, resolveRoomInputStopUrls } from '../lib/session-stop.ts'; +import { + isLiveKitRoomNotFoundError, + resolveLiveKitHttpUrl, + resolveRoomInputStopUrls, +} from '../lib/session-stop.ts'; test('parses the latest target agent worker state from LiveKit server logs', () => { const source = [ @@ -21,6 +25,35 @@ test('maps livekit websocket URLs to server API URLs', () => { assert.equal(resolveLiveKitHttpUrl('https://livekit.example'), 'https://livekit.example'); }); +test('recognizes only LiveKit room-not-found errors as an idempotent stop', () => { + assert.equal( + isLiveKitRoomNotFoundError({ + status: 404, + code: 'not_found', + message: 'requested room does not exist', + }), + true + ); + assert.equal(isLiveKitRoomNotFoundError({ status: 404, code: 'permission_denied' }), false); + assert.equal(isLiveKitRoomNotFoundError({ status: 500, code: 'not_found' }), false); + assert.equal(isLiveKitRoomNotFoundError(new Error('requested room does not exist')), false); +}); + +test('session stop treats an already deleted LiveKit room as stopped', async () => { + const routeSource = await readFile( + new URL('../app/api/session/stop/route.ts', import.meta.url), + 'utf8' + ); + const deleteRoomSource = routeSource.match(/async function deleteLiveKitRoom[\s\S]*?\n}/)?.[0]; + + assert.ok(deleteRoomSource, 'deleteLiveKitRoom should be defined'); + assert.match(deleteRoomSource, /isLiveKitRoomNotFoundError\(error\)/); + assert.match( + deleteRoomSource, + /target:\s*'livekit_room',\s*ok:\s*true,\s*skipped:\s*true,\s*status:\s*404/ + ); +}); + test('room input stop URL resolver ignores primebot non-server input', () => { assert.deepEqual( resolveRoomInputStopUrls({