preact
This commit is contained in:
parent
6acf267d48
commit
b7614eb3f8
4794 changed files with 1280808 additions and 1546 deletions
453
frontend/src/AgentVisualizer.ts
Normal file
453
frontend/src/AgentVisualizer.ts
Normal file
|
|
@ -0,0 +1,453 @@
|
|||
import * as THREE from "three";
|
||||
|
||||
export type VisualizerState = "idle" | "listening" | "thinking" | "speaking";
|
||||
|
||||
export interface AgentVisualizerHandle {
|
||||
setAudioLevel(level: number): void;
|
||||
setState(state: VisualizerState): void;
|
||||
setConnected(connected: boolean): void;
|
||||
setConnecting(connecting: boolean): void;
|
||||
destroy(): void;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Geometry
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function createParaboloidRing(
|
||||
radius = 1.1,
|
||||
segments = 320,
|
||||
tubeRadius = 0.007,
|
||||
): THREE.BufferGeometry {
|
||||
const points: THREE.Vector3[] = [];
|
||||
for (let i = 0; i <= segments; i++) {
|
||||
const theta = (i / segments) * Math.PI * 2;
|
||||
points.push(new THREE.Vector3(radius * Math.cos(theta), 0, radius * Math.sin(theta)));
|
||||
}
|
||||
const curve = new THREE.CatmullRomCurve3(points, true);
|
||||
const geo = new THREE.TubeGeometry(curve, segments, tubeRadius, 12, true);
|
||||
|
||||
const posCount = geo.attributes.position.count;
|
||||
const tAttr = new Float32Array(posCount);
|
||||
const tubularSegments = segments;
|
||||
const radialSegments = 12;
|
||||
for (let tube = 0; tube <= tubularSegments; tube++) {
|
||||
const tVal = tube / tubularSegments;
|
||||
for (let rad = 0; rad <= radialSegments; rad++) {
|
||||
tAttr[tube * (radialSegments + 1) + rad] = tVal;
|
||||
}
|
||||
}
|
||||
geo.setAttribute("aRingT", new THREE.BufferAttribute(tAttr, 1));
|
||||
return geo;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shaders
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ringVertexShader = `
|
||||
attribute float aRingT;
|
||||
uniform float uPhase;
|
||||
uniform float uAmplitude;
|
||||
varying float vWorldX;
|
||||
void main() {
|
||||
float theta = aRingT * 6.28318530718;
|
||||
vec3 pos = position;
|
||||
pos.y += uAmplitude * cos(5.0 * theta + uPhase);
|
||||
vec4 wp = modelMatrix * vec4(pos, 1.0);
|
||||
vWorldX = wp.x;
|
||||
gl_Position = projectionMatrix * viewMatrix * wp;
|
||||
}
|
||||
`;
|
||||
|
||||
const ringFragmentShader = `
|
||||
uniform vec3 uColor;
|
||||
uniform float uFade;
|
||||
uniform float uFadeOffset;
|
||||
varying float vWorldX;
|
||||
void main() {
|
||||
float alpha = 1.0 - uFade * smoothstep(0.0, 1.0, (-vWorldX + uFadeOffset * 1.1) / 1.1 * 0.5 + 0.5);
|
||||
gl_FragColor = vec4(uColor, alpha);
|
||||
}
|
||||
`;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Frame update helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface FrameState {
|
||||
currentAudioLevel: number;
|
||||
smoothAudioLevel: number;
|
||||
smoothAudioLevel2: number;
|
||||
deformScale: number;
|
||||
ringScale: number;
|
||||
spinSpeed: number;
|
||||
cardColorT: number;
|
||||
connectedT: number;
|
||||
prevCardRGB: string;
|
||||
targetConnected: number;
|
||||
isConnecting: boolean;
|
||||
currentState: VisualizerState;
|
||||
}
|
||||
|
||||
interface RingPair {
|
||||
mat1: THREE.ShaderMaterial;
|
||||
mat2: THREE.ShaderMaterial;
|
||||
}
|
||||
|
||||
const CARD_GRAY_RGB = [232, 228, 224];
|
||||
const CARD_IDLE_RGB = [212, 85, 63];
|
||||
const CARD_LISTEN_RGB = [120, 40, 28];
|
||||
|
||||
function updateAudio(s: FrameState, t: number): void {
|
||||
const lerpAudio = 1 - 0.85 ** t;
|
||||
const lerpAudio2 = 1 - 0.94 ** t;
|
||||
s.smoothAudioLevel += (s.currentAudioLevel - s.smoothAudioLevel) * lerpAudio;
|
||||
s.smoothAudioLevel2 += (s.currentAudioLevel - s.smoothAudioLevel2) * lerpAudio2;
|
||||
}
|
||||
|
||||
interface SceneRefs {
|
||||
group: THREE.Group;
|
||||
camera: THREE.OrthographicCamera;
|
||||
speakingSideView: THREE.Vector3;
|
||||
topView: THREE.Vector3;
|
||||
lookAt: THREE.Vector3;
|
||||
applyFrustum: () => void;
|
||||
orthoScaleRef: { value: number };
|
||||
}
|
||||
|
||||
function updateGeometry(s: FrameState, refs: SceneRefs, t: number, now: number): void {
|
||||
const speaking = s.currentState === "speaking";
|
||||
const { group, camera, speakingSideView, topView, lookAt, applyFrustum, orthoScaleRef } = refs;
|
||||
|
||||
let targetDeformScale = 1.0 + s.smoothAudioLevel * 1.1;
|
||||
if (speaking) {
|
||||
targetDeformScale = 2.05 + s.smoothAudioLevel * 2.9;
|
||||
} else if (s.currentState === "thinking") {
|
||||
targetDeformScale = 0.55 + s.smoothAudioLevel * 0.35;
|
||||
}
|
||||
s.deformScale += (targetDeformScale - s.deformScale) * (1 - 0.88 ** t);
|
||||
group.scale.y = s.deformScale;
|
||||
|
||||
const targetRingScale =
|
||||
s.currentState === "thinking"
|
||||
? 1.0 + 0.18 * (0.5 + 0.5 * Math.sin((now * (Math.PI * 2)) / 1000))
|
||||
: 1.0;
|
||||
s.ringScale += (targetRingScale - s.ringScale) * (1 - 0.9 ** t);
|
||||
group.scale.x = s.ringScale;
|
||||
group.scale.z = s.ringScale;
|
||||
|
||||
const targetSpinSpeed = speaking
|
||||
? 0.012 + s.smoothAudioLevel * 0.105
|
||||
: s.currentState === "thinking"
|
||||
? 0.006
|
||||
: 0.0022;
|
||||
s.spinSpeed += (targetSpinSpeed - s.spinSpeed) * (1 - 0.86 ** t);
|
||||
group.rotation.y += s.spinSpeed * t;
|
||||
|
||||
if (speaking || camera.position.distanceToSquared(topView) > 0.0001) {
|
||||
const lerpCamera = 1 - 0.96 ** t;
|
||||
const targetPos = speaking ? speakingSideView : topView;
|
||||
camera.position.lerp(targetPos, lerpCamera);
|
||||
camera.lookAt(lookAt);
|
||||
}
|
||||
|
||||
const camLen = camera.position.length();
|
||||
const sideT = camLen > 0.001 ? Math.abs(camera.position.x) / camLen : 0;
|
||||
const targetOrthoScale = 1.0 - sideT * 0.3;
|
||||
if (Math.abs(targetOrthoScale - orthoScaleRef.value) > 0.0001) {
|
||||
orthoScaleRef.value += (targetOrthoScale - orthoScaleRef.value) * (1 - 0.88 ** t);
|
||||
applyFrustum();
|
||||
}
|
||||
}
|
||||
|
||||
function updateShaderUniforms(
|
||||
s: FrameState,
|
||||
rings: RingPair,
|
||||
camera: THREE.OrthographicCamera,
|
||||
t: number,
|
||||
now: number,
|
||||
): void {
|
||||
const orthoCamera = camera;
|
||||
const camLen = orthoCamera.position.length();
|
||||
const sideT = camLen > 0.001 ? Math.abs(orthoCamera.position.x) / camLen : 0;
|
||||
const lerpSide = 1 - 0.88 ** t;
|
||||
const lerpAmp = 1 - 0.88 ** t;
|
||||
const baseAmp = 0.06 * 1.1;
|
||||
const speaking = s.currentState === "speaking";
|
||||
|
||||
rings.mat1.uniforms.uFade.value += (sideT - rings.mat1.uniforms.uFade.value) * lerpSide;
|
||||
rings.mat1.uniforms.uFadeOffset.value +=
|
||||
(sideT * 0.8 - rings.mat1.uniforms.uFadeOffset.value) * lerpSide;
|
||||
rings.mat2.uniforms.uFade.value += (sideT - rings.mat2.uniforms.uFade.value) * lerpSide;
|
||||
rings.mat2.uniforms.uFadeOffset.value +=
|
||||
(sideT * 0.8 - rings.mat2.uniforms.uFadeOffset.value) * lerpSide;
|
||||
|
||||
if (speaking) {
|
||||
const breathe = Math.sin(now * 0.0018);
|
||||
const base = 1.8 + s.smoothAudioLevel * 4.0;
|
||||
rings.mat1.uniforms.uPhase.value += (base + breathe * 0.6) * (t / 60);
|
||||
rings.mat2.uniforms.uPhase.value +=
|
||||
(base - breathe * 1.4 + s.smoothAudioLevel * 2.0) * (t / 60);
|
||||
const targetAmp1 = baseAmp * (1.0 + s.smoothAudioLevel * 3.5);
|
||||
const targetAmp2 = baseAmp * (1.0 + s.smoothAudioLevel2 * 3.5);
|
||||
rings.mat1.uniforms.uAmplitude.value +=
|
||||
(targetAmp1 - rings.mat1.uniforms.uAmplitude.value) * lerpAmp;
|
||||
rings.mat2.uniforms.uAmplitude.value +=
|
||||
(targetAmp2 - rings.mat2.uniforms.uAmplitude.value) * lerpAmp;
|
||||
} else {
|
||||
rings.mat2.uniforms.uPhase.value +=
|
||||
(Math.PI - rings.mat2.uniforms.uPhase.value) * (1 - 0.92 ** t);
|
||||
rings.mat1.uniforms.uAmplitude.value +=
|
||||
(baseAmp - rings.mat1.uniforms.uAmplitude.value) * lerpAmp;
|
||||
rings.mat2.uniforms.uAmplitude.value +=
|
||||
(baseAmp - rings.mat2.uniforms.uAmplitude.value) * lerpAmp;
|
||||
}
|
||||
}
|
||||
|
||||
function updateCardColor(
|
||||
s: FrameState,
|
||||
renderer: THREE.WebGLRenderer,
|
||||
t: number,
|
||||
now: number,
|
||||
): void {
|
||||
s.connectedT += (s.targetConnected - s.connectedT) * (1 - 0.88 ** t);
|
||||
const throb =
|
||||
s.isConnecting && s.targetConnected === 0
|
||||
? 0.22 * (0.5 - 0.5 * Math.sin((now * (Math.PI * 2)) / 1000))
|
||||
: 0.0;
|
||||
|
||||
const baseR = Math.round(
|
||||
CARD_GRAY_RGB[0] +
|
||||
(CARD_IDLE_RGB[0] - CARD_GRAY_RGB[0]) * s.connectedT -
|
||||
throb * CARD_GRAY_RGB[0],
|
||||
);
|
||||
const baseG = Math.round(
|
||||
CARD_GRAY_RGB[1] +
|
||||
(CARD_IDLE_RGB[1] - CARD_GRAY_RGB[1]) * s.connectedT -
|
||||
throb * CARD_GRAY_RGB[1],
|
||||
);
|
||||
const baseB = Math.round(
|
||||
CARD_GRAY_RGB[2] +
|
||||
(CARD_IDLE_RGB[2] - CARD_GRAY_RGB[2]) * s.connectedT -
|
||||
throb * CARD_GRAY_RGB[2],
|
||||
);
|
||||
|
||||
const targetCardT = s.currentState === "listening" ? 1.0 : 0.0;
|
||||
const cardBase = targetCardT > s.cardColorT ? 0.05 : 0.7;
|
||||
s.cardColorT += (targetCardT - s.cardColorT) * (1 - cardBase ** t);
|
||||
|
||||
const r = Math.min(255, Math.round(baseR + (CARD_LISTEN_RGB[0] - baseR) * s.cardColorT));
|
||||
const g = Math.min(255, Math.round(baseG + (CARD_LISTEN_RGB[1] - baseG) * s.cardColorT));
|
||||
const b = Math.min(255, Math.round(baseB + (CARD_LISTEN_RGB[2] - baseB) * s.cardColorT));
|
||||
const cardRGB = `${r},${g},${b}`;
|
||||
if (cardRGB !== s.prevCardRGB) {
|
||||
renderer.setClearColor((r << 16) | (g << 8) | b, 1);
|
||||
s.prevCardRGB = cardRGB;
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public factory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface SceneSetup {
|
||||
scene: THREE.Scene;
|
||||
camera: THREE.OrthographicCamera;
|
||||
group: THREE.Group;
|
||||
rings: RingPair;
|
||||
lookAt: THREE.Vector3;
|
||||
speakingSideView: THREE.Vector3;
|
||||
topView: THREE.Vector3;
|
||||
}
|
||||
|
||||
function buildScene(): SceneSetup {
|
||||
const scene = new THREE.Scene();
|
||||
const orthoSize = 2.0;
|
||||
const camera = new THREE.OrthographicCamera(
|
||||
-orthoSize,
|
||||
orthoSize,
|
||||
orthoSize,
|
||||
-orthoSize,
|
||||
0.1,
|
||||
40,
|
||||
);
|
||||
const lookAt = new THREE.Vector3(0, 0, 0);
|
||||
const speakingSideView = new THREE.Vector3(3.45, 0, 0);
|
||||
const topView = new THREE.Vector3(0, 3.25, 0.001);
|
||||
camera.position.copy(topView);
|
||||
camera.lookAt(lookAt);
|
||||
scene.add(new THREE.AmbientLight(0xffffff, 1.0));
|
||||
|
||||
const makeRingMaterial = (phase: number) =>
|
||||
new THREE.ShaderMaterial({
|
||||
uniforms: {
|
||||
uColor: { value: new THREE.Color(0xfff5eb) },
|
||||
uFade: { value: 0.0 },
|
||||
uFadeOffset: { value: 0.0 },
|
||||
uPhase: { value: phase },
|
||||
uAmplitude: { value: 0.06 * 1.1 },
|
||||
},
|
||||
vertexShader: ringVertexShader,
|
||||
fragmentShader: ringFragmentShader,
|
||||
transparent: true,
|
||||
side: THREE.DoubleSide,
|
||||
depthWrite: false,
|
||||
});
|
||||
|
||||
const mat1 = makeRingMaterial(0.0);
|
||||
const mat2 = makeRingMaterial(Math.PI);
|
||||
const rings: RingPair = { mat1, mat2 };
|
||||
|
||||
const group = new THREE.Group();
|
||||
group.add(new THREE.Mesh(createParaboloidRing(), mat1));
|
||||
group.add(new THREE.Mesh(createParaboloidRing(), mat2));
|
||||
group.rotation.y = Math.PI * 0.18;
|
||||
scene.add(group);
|
||||
|
||||
return { scene, camera, group, rings, lookAt, speakingSideView, topView };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Frame state factory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeFrameState(): FrameState {
|
||||
return {
|
||||
currentAudioLevel: 0,
|
||||
smoothAudioLevel: 0,
|
||||
smoothAudioLevel2: 0,
|
||||
deformScale: 1.0,
|
||||
ringScale: 1.0,
|
||||
spinSpeed: 0.0,
|
||||
cardColorT: 0.0,
|
||||
connectedT: 0.0,
|
||||
prevCardRGB: "",
|
||||
targetConnected: 0.0,
|
||||
isConnecting: false,
|
||||
currentState: "idle",
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Render loop
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface RenderLoopOpts {
|
||||
renderer: THREE.WebGLRenderer;
|
||||
scene: THREE.Scene;
|
||||
camera: THREE.OrthographicCamera;
|
||||
frameState: FrameState;
|
||||
sceneRefs: SceneRefs;
|
||||
rings: RingPair;
|
||||
}
|
||||
|
||||
function startRenderLoop(opts: RenderLoopOpts): () => void {
|
||||
const { renderer, scene, camera, frameState, sceneRefs, rings } = opts;
|
||||
let lastNow = 0;
|
||||
let rafId = 0;
|
||||
let destroyed = false;
|
||||
|
||||
const renderFrame = (now = 0) => {
|
||||
if (destroyed) return;
|
||||
const dt = Math.min((now - lastNow) / 1000, 0.1);
|
||||
lastNow = now;
|
||||
const t = dt * 60;
|
||||
updateAudio(frameState, t);
|
||||
updateGeometry(frameState, sceneRefs, t, now);
|
||||
updateShaderUniforms(frameState, rings, camera, t, now);
|
||||
updateCardColor(frameState, renderer, t, now);
|
||||
renderer.render(scene, camera);
|
||||
rafId = requestAnimationFrame(renderFrame);
|
||||
};
|
||||
rafId = requestAnimationFrame(renderFrame);
|
||||
|
||||
return () => {
|
||||
destroyed = true;
|
||||
cancelAnimationFrame(rafId);
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public factory
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function createAgentVisualizer(container: HTMLElement): AgentVisualizerHandle {
|
||||
const renderer = new THREE.WebGLRenderer({
|
||||
antialias: true,
|
||||
alpha: false,
|
||||
powerPreference: "high-performance",
|
||||
});
|
||||
renderer.setPixelRatio(1);
|
||||
renderer.setClearColor(0xe8e4e0, 1);
|
||||
renderer.domElement.dataset.ptt = "1";
|
||||
container.innerHTML = "";
|
||||
container.appendChild(renderer.domElement);
|
||||
|
||||
const { scene, camera, group, rings, lookAt, speakingSideView, topView } = buildScene();
|
||||
const orthoSize = 2.0;
|
||||
const orthoScaleRef = { value: 1.0 };
|
||||
|
||||
const applyFrustum = () => {
|
||||
const width = Math.max(2, container.clientWidth);
|
||||
const height = Math.max(2, container.clientHeight);
|
||||
const aspect = width / height;
|
||||
const s = orthoSize * orthoScaleRef.value;
|
||||
if (aspect >= 1) {
|
||||
camera.left = -s * aspect;
|
||||
camera.right = s * aspect;
|
||||
camera.top = s;
|
||||
camera.bottom = -s;
|
||||
} else {
|
||||
camera.left = -s;
|
||||
camera.right = s;
|
||||
camera.top = s / aspect;
|
||||
camera.bottom = -s / aspect;
|
||||
}
|
||||
camera.updateProjectionMatrix();
|
||||
};
|
||||
|
||||
const resize = () => {
|
||||
renderer.setSize(
|
||||
Math.max(2, container.clientWidth),
|
||||
Math.max(2, container.clientHeight),
|
||||
false,
|
||||
);
|
||||
applyFrustum();
|
||||
};
|
||||
resize();
|
||||
window.addEventListener("resize", resize);
|
||||
|
||||
const sceneRefs: SceneRefs = {
|
||||
group,
|
||||
camera,
|
||||
speakingSideView,
|
||||
topView,
|
||||
lookAt,
|
||||
applyFrustum,
|
||||
orthoScaleRef,
|
||||
};
|
||||
const frameState = makeFrameState();
|
||||
const stopLoop = startRenderLoop({ renderer, scene, camera, frameState, sceneRefs, rings });
|
||||
|
||||
return {
|
||||
setAudioLevel(level: number) {
|
||||
frameState.currentAudioLevel = Math.max(0, Math.min(1, Number(level) || 0));
|
||||
},
|
||||
setState(state: VisualizerState) {
|
||||
frameState.currentState = state;
|
||||
},
|
||||
setConnected(connected: boolean) {
|
||||
frameState.targetConnected = connected ? 1.0 : 0.0;
|
||||
if (connected) frameState.isConnecting = false;
|
||||
},
|
||||
setConnecting(connecting: boolean) {
|
||||
frameState.isConnecting = !!connecting;
|
||||
},
|
||||
destroy() {
|
||||
stopLoop();
|
||||
window.removeEventListener("resize", resize);
|
||||
renderer.dispose();
|
||||
},
|
||||
};
|
||||
}
|
||||
61
frontend/src/App.tsx
Normal file
61
frontend/src/App.tsx
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
import { useCallback, useEffect } from "preact/hooks";
|
||||
import { AgentIndicator } from "./components/AgentIndicator";
|
||||
import { ControlBar, VoiceStatus } from "./components/Controls";
|
||||
import { LogPanel } from "./components/LogPanel";
|
||||
import { ToastContainer } from "./components/Toast";
|
||||
import { useAudioMeter } from "./hooks/useAudioMeter";
|
||||
import { usePTT } from "./hooks/usePTT";
|
||||
import { useWebRTC } from "./hooks/useWebRTC";
|
||||
|
||||
export function App() {
|
||||
const rtc = useWebRTC();
|
||||
const audioLevel = useAudioMeter(rtc.remoteStream);
|
||||
|
||||
const { agentStateOverride, handlePointerDown, handlePointerUp } = usePTT({
|
||||
connected: rtc.connected,
|
||||
onSendPtt: (pressed) => rtc.sendJson({ type: "voice-ptt", pressed }),
|
||||
onBootstrap: rtc.connect,
|
||||
});
|
||||
|
||||
const effectiveAgentState = agentStateOverride ?? rtc.agentState;
|
||||
|
||||
useEffect(() => {
|
||||
document.addEventListener("pointerdown", handlePointerDown, { passive: false });
|
||||
document.addEventListener("pointerup", handlePointerUp, { passive: false });
|
||||
document.addEventListener("pointercancel", handlePointerUp, { passive: false });
|
||||
return () => {
|
||||
document.removeEventListener("pointerdown", handlePointerDown);
|
||||
document.removeEventListener("pointerup", handlePointerUp);
|
||||
document.removeEventListener("pointercancel", handlePointerUp);
|
||||
};
|
||||
}, [handlePointerDown, handlePointerUp]);
|
||||
|
||||
const handleReset = useCallback(async () => {
|
||||
await rtc.connect();
|
||||
rtc.sendJson({ type: "command", command: "reset" });
|
||||
}, [rtc]);
|
||||
|
||||
const handleChoice = useCallback(
|
||||
(requestId: string, value: string) => {
|
||||
rtc.sendJson({ type: "ui-response", request_id: requestId, value });
|
||||
},
|
||||
[rtc],
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
<ControlBar onReset={handleReset} />
|
||||
<LogPanel lines={rtc.logLines} />
|
||||
<AgentIndicator
|
||||
state={effectiveAgentState}
|
||||
connected={rtc.connected}
|
||||
connecting={rtc.connecting}
|
||||
audioLevel={audioLevel}
|
||||
onPointerDown={() => {}}
|
||||
onPointerUp={() => {}}
|
||||
/>
|
||||
<VoiceStatus text={rtc.voiceStatus} visible={rtc.statusVisible} />
|
||||
<ToastContainer toasts={rtc.toasts} onDismiss={rtc.dismissToast} onChoice={handleChoice} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
103
frontend/src/audioMeter.ts
Normal file
103
frontend/src/audioMeter.ts
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
const AudioContextCtor =
|
||||
typeof window !== "undefined"
|
||||
? window.AudioContext ||
|
||||
(window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
|
||||
: undefined;
|
||||
|
||||
export interface AudioMeter {
|
||||
connect(stream: MediaStream): void;
|
||||
getLevel(): number;
|
||||
destroy(): void;
|
||||
}
|
||||
|
||||
export function createAudioMeter(): AudioMeter {
|
||||
let audioContext: AudioContext | null = null;
|
||||
let analyser: AnalyserNode | null = null;
|
||||
let sourceNode: MediaStreamAudioSourceNode | null = null;
|
||||
let waveform: Uint8Array<ArrayBuffer> | null = null;
|
||||
let connectedStream: MediaStream | null = null;
|
||||
let running = false;
|
||||
let currentLevel = 0;
|
||||
|
||||
const tick = () => {
|
||||
if (!running) return;
|
||||
if (analyser && waveform) {
|
||||
analyser.getByteTimeDomainData(waveform);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < waveform.length; i++) {
|
||||
const v = (waveform[i] - 128) / 128;
|
||||
sum += v * v;
|
||||
}
|
||||
currentLevel = Math.min(1, Math.sqrt(sum / waveform.length) * 4.8);
|
||||
}
|
||||
requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
const ensureContext = async () => {
|
||||
if (!AudioContextCtor) return;
|
||||
if (!audioContext) {
|
||||
audioContext = new AudioContextCtor();
|
||||
}
|
||||
if (audioContext.state === "suspended") {
|
||||
try {
|
||||
await audioContext.resume();
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
if (!analyser) {
|
||||
analyser = audioContext.createAnalyser();
|
||||
analyser.fftSize = 512;
|
||||
analyser.smoothingTimeConstant = 0.84;
|
||||
waveform = new Uint8Array(analyser.fftSize);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
async connect(stream: MediaStream) {
|
||||
await ensureContext();
|
||||
if (!audioContext || !analyser) return;
|
||||
if (connectedStream === stream) return;
|
||||
if (sourceNode) {
|
||||
try {
|
||||
sourceNode.disconnect();
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
sourceNode = null;
|
||||
}
|
||||
try {
|
||||
sourceNode = audioContext.createMediaStreamSource(stream);
|
||||
sourceNode.connect(analyser);
|
||||
connectedStream = stream;
|
||||
} catch (_) {
|
||||
sourceNode = null;
|
||||
connectedStream = null;
|
||||
}
|
||||
if (!running) {
|
||||
running = true;
|
||||
requestAnimationFrame(tick);
|
||||
}
|
||||
},
|
||||
getLevel() {
|
||||
return currentLevel;
|
||||
},
|
||||
destroy() {
|
||||
running = false;
|
||||
if (sourceNode) {
|
||||
try {
|
||||
sourceNode.disconnect();
|
||||
} catch (_) {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
audioContext?.close().catch(() => {});
|
||||
audioContext = null;
|
||||
analyser = null;
|
||||
sourceNode = null;
|
||||
waveform = null;
|
||||
connectedStream = null;
|
||||
currentLevel = 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
64
frontend/src/components/AgentIndicator.tsx
Normal file
64
frontend/src/components/AgentIndicator.tsx
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
import { useEffect, useRef } from "preact/hooks";
|
||||
import { type AgentVisualizerHandle, createAgentVisualizer } from "../AgentVisualizer";
|
||||
import type { AgentState } from "../types";
|
||||
|
||||
interface Props {
|
||||
state: AgentState;
|
||||
connected: boolean;
|
||||
connecting: boolean;
|
||||
audioLevel: number;
|
||||
onPointerDown(): void;
|
||||
onPointerUp(): void;
|
||||
}
|
||||
|
||||
export function AgentIndicator({
|
||||
state,
|
||||
connected,
|
||||
connecting,
|
||||
audioLevel,
|
||||
onPointerDown,
|
||||
onPointerUp,
|
||||
}: Props) {
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const vizRef = useRef<AgentVisualizerHandle | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!containerRef.current) return;
|
||||
const viz = createAgentVisualizer(containerRef.current);
|
||||
vizRef.current = viz;
|
||||
return () => {
|
||||
viz.destroy();
|
||||
vizRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
vizRef.current?.setState(state);
|
||||
}, [state]);
|
||||
|
||||
useEffect(() => {
|
||||
vizRef.current?.setConnected(connected);
|
||||
}, [connected]);
|
||||
|
||||
useEffect(() => {
|
||||
vizRef.current?.setConnecting(connecting);
|
||||
}, [connecting]);
|
||||
|
||||
useEffect(() => {
|
||||
vizRef.current?.setAudioLevel(audioLevel);
|
||||
}, [audioLevel]);
|
||||
|
||||
return (
|
||||
<div id="agentIndicator" class={`agentIndicator visible ${state}`} data-ptt="1">
|
||||
<div
|
||||
id="agentViz"
|
||||
class="agentViz"
|
||||
data-ptt="1"
|
||||
ref={containerRef}
|
||||
onPointerDown={onPointerDown}
|
||||
onPointerUp={onPointerUp}
|
||||
onPointerCancel={onPointerUp}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
35
frontend/src/components/Controls.tsx
Normal file
35
frontend/src/components/Controls.tsx
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
interface VoiceStatusProps {
|
||||
text: string;
|
||||
visible: boolean;
|
||||
}
|
||||
|
||||
export function VoiceStatus({ text, visible }: VoiceStatusProps) {
|
||||
return (
|
||||
<div id="voiceStatus" class={visible ? "visible" : ""}>
|
||||
{text}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface ControlBarProps {
|
||||
onReset(): void;
|
||||
}
|
||||
|
||||
export function ControlBar({ onReset }: ControlBarProps) {
|
||||
return (
|
||||
<div id="controls">
|
||||
<button
|
||||
id="resetSessionBtn"
|
||||
class="control-btn"
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
onReset();
|
||||
}}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
38
frontend/src/components/LogPanel.tsx
Normal file
38
frontend/src/components/LogPanel.tsx
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
import { useEffect, useRef } from "preact/hooks";
|
||||
import type { LogLine } from "../types";
|
||||
|
||||
interface Props {
|
||||
lines: LogLine[];
|
||||
}
|
||||
|
||||
export function LogPanel({ lines }: Props) {
|
||||
const innerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Scroll to top (newest line — column-reverse layout) after each update
|
||||
useEffect(() => {
|
||||
const el = innerRef.current?.parentElement;
|
||||
if (el) el.scrollTop = 0;
|
||||
}, [lines]);
|
||||
|
||||
return (
|
||||
<div id="log">
|
||||
<div id="log-inner" ref={innerRef}>
|
||||
{lines.map((line) => {
|
||||
const time = line.timestamp ? new Date(line.timestamp).toLocaleTimeString() : "";
|
||||
const role = line.role.trim().toLowerCase();
|
||||
let text: string;
|
||||
if (role === "nanobot") {
|
||||
text = `[${time}] ${line.text.replace(/^(?:nanobot|napbot)\b\s*[:>-]?\s*/i, "")}`;
|
||||
} else {
|
||||
text = `[${time}] ${line.role}: ${line.text}`;
|
||||
}
|
||||
return (
|
||||
<div key={line.id} class={`line ${line.role}`}>
|
||||
{text}
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
109
frontend/src/components/Toast.tsx
Normal file
109
frontend/src/components/Toast.tsx
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { marked } from "marked";
|
||||
import { useEffect, useRef, useState } from "preact/hooks";
|
||||
import type { ToastItem } from "../types";
|
||||
|
||||
interface ToastProps {
|
||||
toast: ToastItem;
|
||||
onDismiss(id: number): void;
|
||||
onChoice(requestId: string, value: string): void;
|
||||
}
|
||||
|
||||
function Toast({ toast, onDismiss, onChoice }: ToastProps) {
|
||||
const [dismissing, setDismissing] = useState(false);
|
||||
const timerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
const dismiss = () => {
|
||||
if (dismissing) return;
|
||||
setDismissing(true);
|
||||
const t = setTimeout(() => onDismiss(toast.id), 400);
|
||||
timerRef.current = t;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (toast.kind !== "choice" && toast.durationMs > 0) {
|
||||
const t = setTimeout(dismiss, toast.durationMs);
|
||||
timerRef.current = t;
|
||||
return () => clearTimeout(t);
|
||||
}
|
||||
return () => {
|
||||
if (timerRef.current) clearTimeout(timerRef.current);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const bodyHtml = (): string => {
|
||||
if (toast.kind === "choice") return "";
|
||||
if (toast.kind === "image") return "";
|
||||
const looksLikeHtml = /^\s*<[a-zA-Z]/.test(toast.content);
|
||||
if (looksLikeHtml) return toast.content;
|
||||
return marked.parse(toast.content) as string;
|
||||
};
|
||||
|
||||
return (
|
||||
<div class={`toast${dismissing ? " dismissing" : ""}`}>
|
||||
<div class="toast-header">
|
||||
{toast.title && <span class="toast-title">{toast.title}</span>}
|
||||
<button
|
||||
class="toast-close"
|
||||
type="button"
|
||||
aria-label="Dismiss"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
dismiss();
|
||||
}}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{toast.kind === "image" && (
|
||||
<img class="toast-image" src={toast.content} alt={toast.title || "image"} />
|
||||
)}
|
||||
|
||||
{toast.kind === "text" && (
|
||||
<div class="toast-body" dangerouslySetInnerHTML={{ __html: bodyHtml() }} />
|
||||
)}
|
||||
|
||||
{toast.kind === "choice" && (
|
||||
<>
|
||||
<div class="toast-body">{toast.question}</div>
|
||||
<div class="toast-choices">
|
||||
{(toast.choices ?? []).map((label) => (
|
||||
<button
|
||||
key={label}
|
||||
class="toast-choice-btn"
|
||||
type="button"
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onChoice(toast.requestId ?? "", label);
|
||||
dismiss();
|
||||
}}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{toast.kind !== "choice" && toast.durationMs > 0 && (
|
||||
<div class="toast-progress" style={{ animationDuration: `${toast.durationMs}ms` }} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface ContainerProps {
|
||||
toasts: ToastItem[];
|
||||
onDismiss(id: number): void;
|
||||
onChoice(requestId: string, value: string): void;
|
||||
}
|
||||
|
||||
export function ToastContainer({ toasts, onDismiss, onChoice }: ContainerProps) {
|
||||
return (
|
||||
<div id="toast-container">
|
||||
{toasts.map((t) => (
|
||||
<Toast key={t.id} toast={t} onDismiss={onDismiss} onChoice={onChoice} />
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
22
frontend/src/hooks/useAudioMeter.ts
Normal file
22
frontend/src/hooks/useAudioMeter.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import { useEffect, useRef, useState } from "preact/hooks";
|
||||
import { createAudioMeter } from "../audioMeter";
|
||||
|
||||
/** Polls the audio level from a MediaStream at rAF rate. Returns a 0–1 level. */
|
||||
export function useAudioMeter(stream: MediaStream | null): number {
|
||||
const meterRef = useRef(createAudioMeter());
|
||||
const [level, setLevel] = useState(0);
|
||||
|
||||
useEffect(() => {
|
||||
const meter = meterRef.current;
|
||||
if (stream) meter.connect(stream);
|
||||
let rafId: number;
|
||||
const tick = () => {
|
||||
setLevel(meter.getLevel());
|
||||
rafId = requestAnimationFrame(tick);
|
||||
};
|
||||
rafId = requestAnimationFrame(tick);
|
||||
return () => cancelAnimationFrame(rafId);
|
||||
}, [stream]);
|
||||
|
||||
return level;
|
||||
}
|
||||
70
frontend/src/hooks/usePTT.ts
Normal file
70
frontend/src/hooks/usePTT.ts
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
import { useCallback, useRef, useState } from "preact/hooks";
|
||||
import type { AgentState } from "../types";
|
||||
|
||||
interface UsePTTOptions {
|
||||
connected: boolean;
|
||||
onSendPtt(pressed: boolean): void;
|
||||
onBootstrap(): Promise<void>;
|
||||
}
|
||||
|
||||
interface PTTState {
|
||||
agentStateOverride: AgentState | null;
|
||||
handlePointerDown(e: Event): Promise<void>;
|
||||
handlePointerUp(e: Event): void;
|
||||
}
|
||||
|
||||
function dispatchMicEnable(enabled: boolean): void {
|
||||
window.dispatchEvent(new CustomEvent("nanobot-mic-enable", { detail: { enabled } }));
|
||||
}
|
||||
|
||||
/** Manages push-to-talk pointer events and mic enable/disable. */
|
||||
export function usePTT({ connected, onSendPtt, onBootstrap }: UsePTTOptions): PTTState {
|
||||
const [agentStateOverride, setAgentStateOverride] = useState<AgentState | null>(null);
|
||||
const activePointers = useRef(new Set<number>());
|
||||
const appStartedRef = useRef(false);
|
||||
|
||||
const beginPTT = useCallback(() => {
|
||||
if (!connected) return;
|
||||
if (agentStateOverride === "listening") return;
|
||||
setAgentStateOverride("listening");
|
||||
dispatchMicEnable(true);
|
||||
onSendPtt(true);
|
||||
}, [connected, agentStateOverride, onSendPtt]);
|
||||
|
||||
const endPTT = useCallback(() => {
|
||||
if (agentStateOverride !== "listening") return;
|
||||
setAgentStateOverride(null);
|
||||
dispatchMicEnable(false);
|
||||
onSendPtt(false);
|
||||
window.dispatchEvent(
|
||||
new CustomEvent("nanobot-show-status", {
|
||||
detail: { text: "Hold anywhere to talk", persistMs: 1800 },
|
||||
}),
|
||||
);
|
||||
}, [agentStateOverride, onSendPtt]);
|
||||
|
||||
const handlePointerDown = useCallback(
|
||||
async (e: Event) => {
|
||||
const pe = e as PointerEvent;
|
||||
if (!(pe.target instanceof Element) || !pe.target.closest("[data-ptt='1']")) return;
|
||||
activePointers.current.add(pe.pointerId);
|
||||
if (!appStartedRef.current) {
|
||||
appStartedRef.current = true;
|
||||
await onBootstrap();
|
||||
}
|
||||
if (activePointers.current.size === 1) beginPTT();
|
||||
},
|
||||
[onBootstrap, beginPTT],
|
||||
);
|
||||
|
||||
const handlePointerUp = useCallback(
|
||||
(e: Event) => {
|
||||
const pe = e as PointerEvent;
|
||||
activePointers.current.delete(pe.pointerId);
|
||||
if (activePointers.current.size === 0) endPTT();
|
||||
},
|
||||
[endPTT],
|
||||
);
|
||||
|
||||
return { agentStateOverride, handlePointerDown, handlePointerUp };
|
||||
}
|
||||
54
frontend/src/hooks/usePushToTalk.ts
Normal file
54
frontend/src/hooks/usePushToTalk.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import { useCallback, useRef, useState } from "preact/hooks";
|
||||
import type { AgentState } from "../types";
|
||||
|
||||
export interface PushToTalkState {
|
||||
pttPressed: boolean;
|
||||
micStream: MediaStream | null;
|
||||
beginPTT(): void;
|
||||
endPTT(): void;
|
||||
}
|
||||
|
||||
interface UsePushToTalkOptions {
|
||||
connected: boolean;
|
||||
agentState: AgentState;
|
||||
onPttChange(pressed: boolean): void;
|
||||
onSetAgentState(state: AgentState): void;
|
||||
onShowStatus(text: string, persistMs?: number): void;
|
||||
}
|
||||
|
||||
export function usePushToTalk({
|
||||
connected,
|
||||
onPttChange,
|
||||
onSetAgentState,
|
||||
onShowStatus,
|
||||
}: UsePushToTalkOptions): PushToTalkState {
|
||||
const [pttPressed, setPttPressed] = useState(false);
|
||||
const micStreamRef = useRef<MediaStream | null>(null);
|
||||
|
||||
// Attach mic stream from RTCPeerConnection tracks — caller passes it via micStream prop
|
||||
// Here we track from the parent. Mic enable/disable is done by the parent hook.
|
||||
|
||||
const beginPTT = useCallback(() => {
|
||||
if (!connected) return;
|
||||
if (pttPressed) return;
|
||||
setPttPressed(true);
|
||||
onPttChange(true);
|
||||
onSetAgentState("listening");
|
||||
onShowStatus("Listening...");
|
||||
}, [connected, pttPressed, onPttChange, onSetAgentState, onShowStatus]);
|
||||
|
||||
const endPTT = useCallback(() => {
|
||||
if (!pttPressed) return;
|
||||
setPttPressed(false);
|
||||
onPttChange(false);
|
||||
onSetAgentState("idle");
|
||||
if (connected) onShowStatus("Hold anywhere to talk", 1800);
|
||||
}, [pttPressed, onPttChange, onSetAgentState, onShowStatus, connected]);
|
||||
|
||||
return {
|
||||
pttPressed,
|
||||
micStream: micStreamRef.current,
|
||||
beginPTT,
|
||||
endPTT,
|
||||
};
|
||||
}
|
||||
466
frontend/src/hooks/useWebRTC.ts
Normal file
466
frontend/src/hooks/useWebRTC.ts
Normal file
|
|
@ -0,0 +1,466 @@
|
|||
import { useCallback, useEffect, useRef, useState } from "preact/hooks";
|
||||
import type { AgentState, ClientMessage, LogLine, ServerMessage, ToastItem } from "../types";
|
||||
|
||||
const BACKEND_URL = import.meta.env.VITE_BACKEND_URL ?? "";
|
||||
|
||||
let toastIdCounter = 0;
|
||||
let logIdCounter = 0;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface WebRTCState {
|
||||
connected: boolean;
|
||||
connecting: boolean;
|
||||
agentState: AgentState;
|
||||
logLines: LogLine[];
|
||||
toasts: ToastItem[];
|
||||
voiceStatus: string;
|
||||
statusVisible: boolean;
|
||||
remoteAudioEl: HTMLAudioElement | null;
|
||||
remoteStream: MediaStream | null;
|
||||
sendJson(msg: ClientMessage): void;
|
||||
dismissToast(id: number): void;
|
||||
connect(): Promise<void>;
|
||||
}
|
||||
|
||||
type AppendLine = (role: string, text: string, timestamp: string) => void;
|
||||
type AddToast = (item: Omit<ToastItem, "id">) => number;
|
||||
type SetAgentState = (updater: (prev: AgentState) => AgentState) => void;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Message handlers (pure functions, outside hook to reduce complexity)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function handleTypedMessage(
|
||||
msg: Extract<ServerMessage, { type: string }>,
|
||||
setAgentState: SetAgentState,
|
||||
appendLine: AppendLine,
|
||||
addToast: AddToast,
|
||||
): void {
|
||||
if (msg.type === "agent_state") {
|
||||
const s = (msg as { type: "agent_state"; state: AgentState }).state;
|
||||
setAgentState((prev) => (prev === "listening" ? prev : s));
|
||||
return;
|
||||
}
|
||||
if (msg.type === "message") {
|
||||
const mm = msg as { type: "message"; content: string; is_progress: boolean };
|
||||
if (!mm.is_progress) appendLine("nanobot", mm.content, "");
|
||||
return;
|
||||
}
|
||||
if (msg.type === "toast") {
|
||||
const tm = msg as {
|
||||
type: "toast";
|
||||
kind: "text" | "image";
|
||||
content: string;
|
||||
title: string;
|
||||
duration_ms: number;
|
||||
};
|
||||
addToast({
|
||||
kind: tm.kind,
|
||||
content: tm.content,
|
||||
title: tm.title,
|
||||
durationMs: tm.duration_ms ?? 6000,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (msg.type === "choice") {
|
||||
const cm = msg as {
|
||||
type: "choice";
|
||||
request_id: string;
|
||||
question: string;
|
||||
choices: string[];
|
||||
title: string;
|
||||
};
|
||||
addToast({
|
||||
kind: "choice",
|
||||
content: "",
|
||||
title: cm.title || "",
|
||||
durationMs: 0,
|
||||
requestId: cm.request_id,
|
||||
question: cm.question,
|
||||
choices: cm.choices,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (msg.type === "error") {
|
||||
appendLine("system", (msg as { type: "error"; error: string }).error, "");
|
||||
}
|
||||
// pong and rtc-* are no-ops
|
||||
}
|
||||
|
||||
function parseLegacyToast(text: string, addToast: AddToast): void {
|
||||
console.log("[toast] parseLegacyToast raw text:", text);
|
||||
try {
|
||||
const t = JSON.parse(text);
|
||||
console.log("[toast] parsed toast object:", t);
|
||||
addToast({
|
||||
kind: t.kind || "text",
|
||||
content: t.content || "",
|
||||
title: t.title || "",
|
||||
durationMs: typeof t.duration_ms === "number" ? t.duration_ms : 6000,
|
||||
});
|
||||
} catch {
|
||||
console.log("[toast] JSON parse failed, using raw text as content");
|
||||
addToast({ kind: "text", content: text, title: "", durationMs: 6000 });
|
||||
}
|
||||
}
|
||||
|
||||
function parseLegacyChoice(text: string, addToast: AddToast): void {
|
||||
try {
|
||||
const c = JSON.parse(text);
|
||||
addToast({
|
||||
kind: "choice",
|
||||
content: "",
|
||||
title: c.title || "",
|
||||
durationMs: 0,
|
||||
requestId: c.request_id || "",
|
||||
question: c.question || "",
|
||||
choices: Array.isArray(c.choices) ? c.choices : [],
|
||||
});
|
||||
} catch {
|
||||
/* ignore malformed */
|
||||
}
|
||||
}
|
||||
|
||||
function handleLegacyMessage(
|
||||
rm: { role: string; text: string; timestamp?: string },
|
||||
setAgentState: SetAgentState,
|
||||
appendLine: AppendLine,
|
||||
addToast: AddToast,
|
||||
): void {
|
||||
const role = (rm.role || "system").toString();
|
||||
const text = (rm.text || "").toString();
|
||||
const ts = rm.timestamp || "";
|
||||
|
||||
if (role === "agent-state") {
|
||||
const newState = text.trim() as AgentState;
|
||||
setAgentState((prev) => (prev === "listening" ? prev : newState));
|
||||
return;
|
||||
}
|
||||
if (role === "toast") {
|
||||
parseLegacyToast(text, addToast);
|
||||
return;
|
||||
}
|
||||
if (role === "choice") {
|
||||
parseLegacyChoice(text, addToast);
|
||||
return;
|
||||
}
|
||||
if (role === "wisper") return; // suppress debug
|
||||
appendLine(role, text, ts);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// WebRTC helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function acquireMicStream(): Promise<MediaStream> {
|
||||
try {
|
||||
return await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
channelCount: 1,
|
||||
sampleRate: 48000,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: false,
|
||||
},
|
||||
video: false,
|
||||
});
|
||||
} catch {
|
||||
return navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
||||
}
|
||||
}
|
||||
|
||||
function waitForIceComplete(pc: RTCPeerConnection): Promise<void> {
|
||||
return new Promise<void>((resolve) => {
|
||||
if (pc.iceGatheringState === "complete") {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
const check = () => {
|
||||
if (pc.iceGatheringState === "complete") {
|
||||
pc.removeEventListener("icegatheringstatechange", check);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
pc.addEventListener("icegatheringstatechange", check);
|
||||
setTimeout(resolve, 5000); // safety timeout
|
||||
});
|
||||
}
|
||||
|
||||
async function exchangeSdp(
|
||||
localDesc: RTCSessionDescription,
|
||||
): Promise<{ sdp: string; rtcType: string }> {
|
||||
const rtcUrl = BACKEND_URL ? `${BACKEND_URL}/rtc/offer` : "/rtc/offer";
|
||||
const resp = await fetch(rtcUrl, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ sdp: localDesc.sdp, rtcType: localDesc.type }),
|
||||
});
|
||||
if (!resp.ok) throw new Error(`/rtc/offer returned ${resp.status}`);
|
||||
return resp.json() as Promise<{ sdp: string; rtcType: string }>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hook internals
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface RTCRefs {
|
||||
pcRef: { current: RTCPeerConnection | null };
|
||||
dcRef: { current: RTCDataChannel | null };
|
||||
remoteAudioRef: { current: HTMLAudioElement | null };
|
||||
micSendersRef: { current: RTCRtpSender[] };
|
||||
}
|
||||
|
||||
interface RTCCallbacks {
|
||||
setConnected: (v: boolean) => void;
|
||||
setConnecting: (v: boolean) => void;
|
||||
setRemoteStream: (s: MediaStream | null) => void;
|
||||
showStatus: (text: string, persistMs?: number) => void;
|
||||
appendLine: AppendLine;
|
||||
onDcMessage: (raw: string) => void;
|
||||
closePC: () => void;
|
||||
}
|
||||
|
||||
async function runConnect(refs: RTCRefs, cbs: RTCCallbacks): Promise<void> {
|
||||
if (refs.pcRef.current) return;
|
||||
if (!window.RTCPeerConnection) {
|
||||
cbs.showStatus("WebRTC unavailable in this browser.", 4000);
|
||||
return;
|
||||
}
|
||||
cbs.setConnecting(true);
|
||||
cbs.showStatus("Connecting...");
|
||||
|
||||
let micStream: MediaStream | null = null;
|
||||
try {
|
||||
micStream = await acquireMicStream();
|
||||
micStream.getAudioTracks().forEach((t) => {
|
||||
t.enabled = false;
|
||||
});
|
||||
|
||||
const pc = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.l.google.com:19302" }] });
|
||||
refs.pcRef.current = pc;
|
||||
|
||||
const newRemoteStream = new MediaStream();
|
||||
cbs.setRemoteStream(newRemoteStream);
|
||||
if (refs.remoteAudioRef.current) {
|
||||
refs.remoteAudioRef.current.srcObject = newRemoteStream;
|
||||
refs.remoteAudioRef.current.play().catch(() => {});
|
||||
}
|
||||
|
||||
pc.ontrack = (event) => {
|
||||
if (event.track.kind !== "audio") return;
|
||||
newRemoteStream.addTrack(event.track);
|
||||
refs.remoteAudioRef.current?.play().catch(() => {});
|
||||
};
|
||||
|
||||
const dc = pc.createDataChannel("app", { ordered: true });
|
||||
refs.dcRef.current = dc;
|
||||
dc.onopen = () => {
|
||||
cbs.setConnected(true);
|
||||
cbs.setConnecting(false);
|
||||
cbs.showStatus("Hold anywhere to talk", 2500);
|
||||
cbs.appendLine("system", "Connected.", new Date().toISOString());
|
||||
};
|
||||
dc.onclose = () => {
|
||||
cbs.appendLine("system", "Disconnected.", new Date().toISOString());
|
||||
cbs.closePC();
|
||||
};
|
||||
dc.onmessage = (e) => cbs.onDcMessage(e.data as string);
|
||||
|
||||
const stream = micStream;
|
||||
stream.getAudioTracks().forEach((track) => {
|
||||
pc.addTrack(track, stream);
|
||||
});
|
||||
refs.micSendersRef.current = pc.getSenders().filter((s) => s.track?.kind === "audio");
|
||||
|
||||
const offer = await pc.createOffer();
|
||||
await pc.setLocalDescription(offer);
|
||||
await waitForIceComplete(pc);
|
||||
|
||||
const localDesc = pc.localDescription;
|
||||
if (!localDesc) throw new Error("No local description after ICE gathering");
|
||||
const answer = await exchangeSdp(localDesc);
|
||||
await pc.setRemoteDescription({ type: answer.rtcType as RTCSdpType, sdp: answer.sdp });
|
||||
} catch (err) {
|
||||
cbs.appendLine("system", `Connection failed: ${err}`, new Date().toISOString());
|
||||
cbs.showStatus("Connection failed.", 3000);
|
||||
cbs.closePC();
|
||||
if (micStream)
|
||||
micStream.getTracks().forEach((t) => {
|
||||
t.stop();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Message state sub-hook
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface MessageState {
|
||||
agentState: AgentState;
|
||||
logLines: LogLine[];
|
||||
toasts: ToastItem[];
|
||||
appendLine: AppendLine;
|
||||
addToast: AddToast;
|
||||
dismissToast: (id: number) => void;
|
||||
onDcMessage: (raw: string) => void;
|
||||
}
|
||||
|
||||
function useMessageState(): MessageState {
|
||||
const [agentState, setAgentState] = useState<AgentState>("idle");
|
||||
const [logLines, setLogLines] = useState<LogLine[]>([]);
|
||||
const [toasts, setToasts] = useState<ToastItem[]>([]);
|
||||
|
||||
const appendLine = useCallback((role: string, text: string, timestamp: string) => {
|
||||
setLogLines((prev) => {
|
||||
const next = [
|
||||
...prev,
|
||||
{ id: logIdCounter++, role, text, timestamp: timestamp || new Date().toISOString() },
|
||||
];
|
||||
return next.length > 250 ? next.slice(next.length - 250) : next;
|
||||
});
|
||||
}, []);
|
||||
|
||||
const addToast = useCallback((item: Omit<ToastItem, "id">) => {
|
||||
const id = toastIdCounter++;
|
||||
setToasts((prev) => [{ ...item, id }, ...prev]);
|
||||
return id;
|
||||
}, []);
|
||||
|
||||
const dismissToast = useCallback((id: number) => {
|
||||
setToasts((prev) => prev.filter((t) => t.id !== id));
|
||||
}, []);
|
||||
|
||||
const onDcMessage = useCallback(
|
||||
(raw: string) => {
|
||||
console.log("[dc] onDcMessage raw:", raw);
|
||||
let msg: ServerMessage;
|
||||
try {
|
||||
msg = JSON.parse(raw);
|
||||
} catch {
|
||||
console.log("[dc] JSON parse failed for raw message");
|
||||
return;
|
||||
}
|
||||
if ("type" in msg) {
|
||||
console.log("[dc] typed message, type:", (msg as { type: string }).type);
|
||||
handleTypedMessage(
|
||||
msg as Extract<ServerMessage, { type: string }>,
|
||||
setAgentState,
|
||||
appendLine,
|
||||
addToast,
|
||||
);
|
||||
} else {
|
||||
console.log("[dc] legacy message, role:", (msg as { role: string }).role);
|
||||
handleLegacyMessage(
|
||||
msg as { role: string; text: string; timestamp?: string },
|
||||
setAgentState,
|
||||
appendLine,
|
||||
addToast,
|
||||
);
|
||||
}
|
||||
},
|
||||
[appendLine, addToast],
|
||||
);
|
||||
|
||||
return { agentState, logLines, toasts, appendLine, addToast, dismissToast, onDcMessage };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Hook
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function useWebRTC(): WebRTCState {
|
||||
const [connected, setConnected] = useState(false);
|
||||
const [connecting, setConnecting] = useState(false);
|
||||
const [voiceStatus, setVoiceStatus] = useState("");
|
||||
const [statusVisible, setStatusVisible] = useState(false);
|
||||
const [remoteStream, setRemoteStream] = useState<MediaStream | null>(null);
|
||||
|
||||
const pcRef = useRef<RTCPeerConnection | null>(null);
|
||||
const dcRef = useRef<RTCDataChannel | null>(null);
|
||||
const remoteAudioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const statusTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const micSendersRef = useRef<RTCRtpSender[]>([]);
|
||||
|
||||
const { agentState, logLines, toasts, appendLine, dismissToast, onDcMessage } = useMessageState();
|
||||
|
||||
// Create audio element once
|
||||
useEffect(() => {
|
||||
const audio = new Audio();
|
||||
audio.autoplay = true;
|
||||
(audio as HTMLAudioElement & { playsInline: boolean }).playsInline = true;
|
||||
remoteAudioRef.current = audio;
|
||||
return () => {
|
||||
audio.srcObject = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const handler = (e: Event) => {
|
||||
const enabled = (e as CustomEvent<{ enabled: boolean }>).detail?.enabled ?? false;
|
||||
micSendersRef.current.forEach((sender) => {
|
||||
if (sender.track) sender.track.enabled = enabled;
|
||||
});
|
||||
};
|
||||
window.addEventListener("nanobot-mic-enable", handler);
|
||||
return () => window.removeEventListener("nanobot-mic-enable", handler);
|
||||
}, []);
|
||||
|
||||
const showStatus = useCallback((text: string, persistMs = 0) => {
|
||||
setVoiceStatus(text);
|
||||
setStatusVisible(true);
|
||||
if (statusTimerRef.current) clearTimeout(statusTimerRef.current);
|
||||
if (persistMs > 0) {
|
||||
statusTimerRef.current = setTimeout(() => setStatusVisible(false), persistMs);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const sendJson = useCallback((msg: ClientMessage) => {
|
||||
const dc = dcRef.current;
|
||||
if (!dc || dc.readyState !== "open") return;
|
||||
dc.send(JSON.stringify(msg));
|
||||
}, []);
|
||||
|
||||
const closePC = useCallback(() => {
|
||||
dcRef.current?.close();
|
||||
dcRef.current = null;
|
||||
pcRef.current?.close();
|
||||
pcRef.current = null;
|
||||
micSendersRef.current = [];
|
||||
setConnected(false);
|
||||
setConnecting(false);
|
||||
if (remoteAudioRef.current) remoteAudioRef.current.srcObject = null;
|
||||
setRemoteStream(null);
|
||||
}, []);
|
||||
|
||||
const connect = useCallback(async () => {
|
||||
const refs: RTCRefs = { pcRef, dcRef, remoteAudioRef, micSendersRef };
|
||||
const cbs: RTCCallbacks = {
|
||||
setConnected,
|
||||
setConnecting,
|
||||
setRemoteStream,
|
||||
showStatus,
|
||||
appendLine,
|
||||
onDcMessage,
|
||||
closePC,
|
||||
};
|
||||
await runConnect(refs, cbs);
|
||||
}, [setConnected, setConnecting, setRemoteStream, showStatus, appendLine, onDcMessage, closePC]);
|
||||
|
||||
return {
|
||||
connected,
|
||||
connecting,
|
||||
agentState,
|
||||
logLines,
|
||||
toasts,
|
||||
voiceStatus,
|
||||
statusVisible,
|
||||
remoteAudioEl: remoteAudioRef.current,
|
||||
remoteStream,
|
||||
sendJson,
|
||||
dismissToast,
|
||||
connect,
|
||||
};
|
||||
}
|
||||
431
frontend/src/index.css
Normal file
431
frontend/src/index.css
Normal file
|
|
@ -0,0 +1,431 @@
|
|||
* {
|
||||
box-sizing: border-box;
|
||||
user-select: none;
|
||||
-webkit-user-select: none;
|
||||
}
|
||||
|
||||
html,
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
overflow: hidden;
|
||||
background: #ffffff;
|
||||
touch-action: none;
|
||||
}
|
||||
|
||||
#app {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
/* --- Log panel --- */
|
||||
#log {
|
||||
position: fixed;
|
||||
bottom: calc(5vh + 20px);
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: calc(90vw - 40px);
|
||||
max-height: 22vh;
|
||||
overflow-y: auto;
|
||||
padding: 12px 14px;
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
color: rgba(30, 20, 10, 0.35);
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
display: flex;
|
||||
flex-direction: column-reverse;
|
||||
border-radius: 10px;
|
||||
background: transparent;
|
||||
transition:
|
||||
color 0.3s,
|
||||
background 0.3s;
|
||||
z-index: 10;
|
||||
pointer-events: auto;
|
||||
-webkit-mask-image: linear-gradient(to top, black 55%, transparent 100%);
|
||||
mask-image: linear-gradient(to top, black 55%, transparent 100%);
|
||||
}
|
||||
#log:hover {
|
||||
color: rgba(30, 20, 10, 0.85);
|
||||
background: rgba(0, 0, 0, 0.06);
|
||||
-webkit-mask-image: none;
|
||||
mask-image: none;
|
||||
}
|
||||
#log * {
|
||||
user-select: text;
|
||||
-webkit-user-select: text;
|
||||
}
|
||||
#log-inner {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.line {
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.line.user {
|
||||
color: rgba(20, 10, 0, 0.85);
|
||||
}
|
||||
.line.system {
|
||||
color: rgba(120, 80, 40, 0.5);
|
||||
}
|
||||
.line.wisper {
|
||||
color: rgba(120, 80, 40, 0.4);
|
||||
}
|
||||
#log:hover .line.user {
|
||||
color: rgba(20, 10, 0, 1);
|
||||
}
|
||||
#log:hover .line.system {
|
||||
color: rgba(120, 80, 40, 0.85);
|
||||
}
|
||||
#log:hover .line.wisper {
|
||||
color: rgba(120, 80, 40, 0.75);
|
||||
}
|
||||
|
||||
/* --- Voice status --- */
|
||||
#voiceStatus {
|
||||
position: fixed;
|
||||
bottom: 12px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: rgba(0, 0, 0, 0.08);
|
||||
color: #111111;
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
padding: 4px 12px;
|
||||
border-radius: 99px;
|
||||
pointer-events: none;
|
||||
white-space: nowrap;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
#voiceStatus.visible {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* --- Agent indicator --- */
|
||||
.agentIndicator {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 18px;
|
||||
pointer-events: none;
|
||||
opacity: 0;
|
||||
transition: opacity 0.4s;
|
||||
}
|
||||
.agentIndicator.visible {
|
||||
opacity: 1;
|
||||
}
|
||||
.agentViz {
|
||||
width: 90vw;
|
||||
height: 90vh;
|
||||
aspect-ratio: unset;
|
||||
border-radius: 24px;
|
||||
box-shadow:
|
||||
0 8px 32px rgba(0, 0, 0, 0.25),
|
||||
4px 4px 0px rgba(0, 0, 0, 0.15);
|
||||
overflow: hidden;
|
||||
pointer-events: auto;
|
||||
cursor: pointer;
|
||||
}
|
||||
.agentViz canvas {
|
||||
/* biome-ignore lint/complexity/noImportantStyles: Three.js sets inline width/height that must be overridden */
|
||||
width: 100% !important;
|
||||
/* biome-ignore lint/complexity/noImportantStyles: Three.js sets inline width/height that must be overridden */
|
||||
height: 100% !important;
|
||||
display: block;
|
||||
pointer-events: auto;
|
||||
}
|
||||
.agentIndicator.idle {
|
||||
color: #6b3a28;
|
||||
}
|
||||
.agentIndicator.listening {
|
||||
color: #d4553f;
|
||||
}
|
||||
.agentIndicator.thinking {
|
||||
color: #a0522d;
|
||||
}
|
||||
.agentIndicator.speaking {
|
||||
color: #8b4513;
|
||||
}
|
||||
|
||||
/* --- Controls --- */
|
||||
#controls {
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
right: 12px;
|
||||
z-index: 20;
|
||||
pointer-events: auto;
|
||||
}
|
||||
.control-btn {
|
||||
border: none;
|
||||
background: #ffffff;
|
||||
color: #111111;
|
||||
border-radius: 10px;
|
||||
padding: 7px 12px;
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
letter-spacing: 0.04em;
|
||||
cursor: pointer;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
.control-btn:active {
|
||||
transform: translateY(1px);
|
||||
box-shadow: 0 1px 4px rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
|
||||
/* --- Toast container --- */
|
||||
#toast-container {
|
||||
position: fixed;
|
||||
top: 16px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: min(92vw, 480px);
|
||||
max-height: calc(100vh - 32px);
|
||||
overflow-y: auto;
|
||||
overflow-x: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
z-index: 100;
|
||||
pointer-events: auto;
|
||||
touch-action: pan-y;
|
||||
-webkit-overflow-scrolling: touch;
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: rgba(255, 200, 140, 0.25) transparent;
|
||||
padding-bottom: 4px;
|
||||
}
|
||||
#toast-container::-webkit-scrollbar {
|
||||
width: 4px;
|
||||
}
|
||||
#toast-container::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
#toast-container::-webkit-scrollbar-thumb {
|
||||
background: rgba(255, 200, 140, 0.25);
|
||||
border-radius: 2px;
|
||||
}
|
||||
|
||||
/* --- Individual toast --- */
|
||||
.toast {
|
||||
pointer-events: auto;
|
||||
background: rgba(28, 22, 16, 0.92);
|
||||
border: 1px solid rgba(255, 200, 140, 0.18);
|
||||
border-radius: 12px;
|
||||
padding: 14px 16px 14px 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
box-shadow: 0 4px 24px rgba(0, 0, 0, 0.45);
|
||||
animation: toast-in 0.22s cubic-bezier(0.34, 1.4, 0.64, 1) both;
|
||||
position: relative;
|
||||
max-width: 100%;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
.toast.dismissing {
|
||||
animation: toast-out 0.18s ease-in both;
|
||||
}
|
||||
@keyframes toast-in {
|
||||
from {
|
||||
opacity: 0;
|
||||
transform: translateY(-14px) scale(0.96);
|
||||
}
|
||||
to {
|
||||
opacity: 1;
|
||||
transform: translateY(0) scale(1);
|
||||
}
|
||||
}
|
||||
@keyframes toast-out {
|
||||
from {
|
||||
opacity: 1;
|
||||
transform: translateY(0) scale(1);
|
||||
}
|
||||
to {
|
||||
opacity: 0;
|
||||
transform: translateY(-10px) scale(0.96);
|
||||
}
|
||||
}
|
||||
.toast-progress {
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
height: 2px;
|
||||
background: rgba(255, 190, 120, 0.55);
|
||||
width: 100%;
|
||||
transform-origin: left;
|
||||
animation: toast-progress-shrink linear both;
|
||||
border-radius: 0 0 12px 12px;
|
||||
}
|
||||
@keyframes toast-progress-shrink {
|
||||
from {
|
||||
transform: scaleX(1);
|
||||
}
|
||||
to {
|
||||
transform: scaleX(0);
|
||||
}
|
||||
}
|
||||
.toast-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-start;
|
||||
gap: 10px;
|
||||
}
|
||||
.toast-title {
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.07em;
|
||||
color: rgba(255, 200, 140, 0.85);
|
||||
text-transform: uppercase;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.toast-close {
|
||||
background: none;
|
||||
border: none;
|
||||
color: rgba(255, 245, 235, 0.35);
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
padding: 0 2px;
|
||||
flex-shrink: 0;
|
||||
transition: color 0.15s;
|
||||
}
|
||||
.toast-close:hover {
|
||||
color: rgba(255, 245, 235, 0.85);
|
||||
}
|
||||
.toast-body {
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.65;
|
||||
color: rgba(255, 245, 235, 0.82);
|
||||
white-space: normal;
|
||||
word-break: break-word;
|
||||
user-select: text;
|
||||
-webkit-user-select: text;
|
||||
}
|
||||
.toast-body p {
|
||||
margin: 0 0 6px;
|
||||
}
|
||||
.toast-body p:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
.toast-body h1,
|
||||
.toast-body h2,
|
||||
.toast-body h3,
|
||||
.toast-body h4,
|
||||
.toast-body h5,
|
||||
.toast-body h6 {
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
color: rgba(255, 200, 140, 0.95);
|
||||
margin: 8px 0 4px;
|
||||
}
|
||||
.toast-body ul,
|
||||
.toast-body ol {
|
||||
margin: 4px 0 6px;
|
||||
padding-left: 18px;
|
||||
}
|
||||
.toast-body li {
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
.toast-body code {
|
||||
background: rgba(255, 255, 255, 0.07);
|
||||
border-radius: 4px;
|
||||
padding: 1px 5px;
|
||||
font-size: 11px;
|
||||
}
|
||||
.toast-body pre {
|
||||
background: rgba(0, 0, 0, 0.35);
|
||||
border-radius: 6px;
|
||||
padding: 8px 10px;
|
||||
overflow-x: auto;
|
||||
margin: 6px 0;
|
||||
}
|
||||
.toast-body pre code {
|
||||
background: none;
|
||||
padding: 0;
|
||||
font-size: 11px;
|
||||
}
|
||||
.toast-body table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
font-size: 11px;
|
||||
margin: 6px 0;
|
||||
}
|
||||
.toast-body th,
|
||||
.toast-body td {
|
||||
border: 1px solid rgba(255, 200, 140, 0.2);
|
||||
padding: 4px 8px;
|
||||
text-align: left;
|
||||
}
|
||||
.toast-body th {
|
||||
background: rgba(255, 200, 140, 0.08);
|
||||
color: rgba(255, 200, 140, 0.9);
|
||||
font-weight: 600;
|
||||
}
|
||||
.toast-body a {
|
||||
color: rgba(255, 200, 140, 0.85);
|
||||
text-decoration: underline;
|
||||
}
|
||||
.toast-body blockquote {
|
||||
border-left: 3px solid rgba(255, 200, 140, 0.3);
|
||||
margin: 6px 0;
|
||||
padding-left: 10px;
|
||||
color: rgba(255, 245, 235, 0.55);
|
||||
}
|
||||
.toast-body hr {
|
||||
border: none;
|
||||
border-top: 1px solid rgba(255, 200, 140, 0.15);
|
||||
margin: 8px 0;
|
||||
}
|
||||
.toast-choices {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
.toast-choice-btn {
|
||||
background: rgba(255, 200, 140, 0.12);
|
||||
border: 1px solid rgba(255, 200, 140, 0.35);
|
||||
border-radius: 8px;
|
||||
color: rgba(255, 245, 235, 0.9);
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
padding: 6px 14px;
|
||||
cursor: pointer;
|
||||
transition:
|
||||
background 0.15s,
|
||||
border-color 0.15s;
|
||||
flex: 1 1 auto;
|
||||
text-align: center;
|
||||
}
|
||||
.toast-choice-btn:hover {
|
||||
background: rgba(255, 200, 140, 0.25);
|
||||
border-color: rgba(255, 200, 140, 0.65);
|
||||
}
|
||||
.toast-choice-btn:active {
|
||||
background: rgba(255, 200, 140, 0.38);
|
||||
}
|
||||
.toast-choice-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
.toast-image {
|
||||
width: 100%;
|
||||
height: auto;
|
||||
object-fit: contain;
|
||||
border-radius: 8px;
|
||||
display: block;
|
||||
}
|
||||
6
frontend/src/main.tsx
Normal file
6
frontend/src/main.tsx
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
import { render } from "preact";
|
||||
import { App } from "./App";
|
||||
import "./index.css";
|
||||
|
||||
const root = document.getElementById("app");
|
||||
if (root) render(<App />, root);
|
||||
45
frontend/src/types.ts
Normal file
45
frontend/src/types.ts
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
// Shared TypeScript types for the Nanobot UI
|
||||
|
||||
export type AgentState = "idle" | "listening" | "thinking" | "speaking";
|
||||
|
||||
// Messages sent FROM backend TO frontend via DataChannel
|
||||
export type ServerMessage =
|
||||
| { type: "agent_state"; state: AgentState }
|
||||
| { type: "message"; content: string; is_progress: boolean }
|
||||
| { type: "toast"; kind: "text" | "image"; content: string; title: string; duration_ms: number }
|
||||
| { type: "choice"; request_id: string; question: string; choices: string[]; title: string }
|
||||
| { type: "error"; error: string }
|
||||
| { type: "pong" }
|
||||
// Legacy wire format still used by backend (role-based)
|
||||
| { role: string; text: string; timestamp?: string };
|
||||
|
||||
// Messages sent FROM frontend TO backend via DataChannel
|
||||
export type ClientMessage =
|
||||
| { type: "voice-ptt"; pressed: boolean }
|
||||
| { type: "command"; command: string }
|
||||
| { type: "ui-response"; request_id: string; value: string }
|
||||
| { type: "ping" };
|
||||
|
||||
export interface LogLine {
|
||||
id: number;
|
||||
role: string;
|
||||
text: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
export interface ToastItem {
|
||||
id: number;
|
||||
kind: "text" | "image" | "choice";
|
||||
content: string;
|
||||
title: string;
|
||||
durationMs: number;
|
||||
// For choice toasts
|
||||
requestId?: string;
|
||||
question?: string;
|
||||
choices?: string[];
|
||||
}
|
||||
|
||||
export interface RTCState {
|
||||
connected: boolean;
|
||||
connecting: boolean;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue