api channel and tools
This commit is contained in:
parent
9222c59f03
commit
3816a9627e
4 changed files with 684 additions and 582 deletions
|
|
@ -16,7 +16,7 @@
|
|||
width: 100%;
|
||||
height: 100%;
|
||||
overflow: hidden;
|
||||
background: #1a1510;
|
||||
background: #ffffff;
|
||||
touch-action: none;
|
||||
}
|
||||
#log {
|
||||
|
|
@ -31,7 +31,7 @@
|
|||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.6;
|
||||
color: rgba(255, 245, 235, 0.35);
|
||||
color: rgba(30, 20, 10, 0.35);
|
||||
white-space: pre-wrap;
|
||||
word-break: break-word;
|
||||
display: flex;
|
||||
|
|
@ -45,8 +45,8 @@
|
|||
mask-image: linear-gradient(to top, black 55%, transparent 100%);
|
||||
}
|
||||
#log:hover {
|
||||
color: rgba(255, 245, 235, 0.92);
|
||||
background: rgba(0, 0, 0, 0.18);
|
||||
color: rgba(30, 20, 10, 0.85);
|
||||
background: rgba(0, 0, 0, 0.06);
|
||||
-webkit-mask-image: none;
|
||||
mask-image: none;
|
||||
}
|
||||
|
|
@ -62,17 +62,17 @@
|
|||
margin-bottom: 4px;
|
||||
}
|
||||
.line.user {
|
||||
color: rgba(255, 255, 255, 0.9);
|
||||
color: rgba(20, 10, 0, 0.85);
|
||||
}
|
||||
.line.system {
|
||||
color: rgba(255, 220, 180, 0.5);
|
||||
color: rgba(120, 80, 40, 0.5);
|
||||
}
|
||||
.line.wisper {
|
||||
color: rgba(255, 200, 160, 0.4);
|
||||
color: rgba(120, 80, 40, 0.4);
|
||||
}
|
||||
#log:hover .line.user { color: rgba(255, 255, 255, 1.0); }
|
||||
#log:hover .line.system { color: rgba(255, 220, 180, 0.85); }
|
||||
#log:hover .line.wisper { color: rgba(255, 200, 160, 0.75); }
|
||||
#log:hover .line.user { color: rgba(20, 10, 0, 1.0); }
|
||||
#log:hover .line.system { color: rgba(120, 80, 40, 0.85); }
|
||||
#log:hover .line.wisper { color: rgba(120, 80, 40, 0.75); }
|
||||
#voiceStatus {
|
||||
position: fixed;
|
||||
bottom: 12px;
|
||||
|
|
@ -119,11 +119,14 @@
|
|||
border-radius: 24px;
|
||||
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.25), 4px 4px 0px rgba(0,0,0,0.15);
|
||||
overflow: hidden;
|
||||
pointer-events: auto;
|
||||
cursor: pointer;
|
||||
}
|
||||
#agentViz canvas {
|
||||
width: 100% !important;
|
||||
height: 100% !important;
|
||||
display: block;
|
||||
pointer-events: auto;
|
||||
}
|
||||
#agentIndicator .label {
|
||||
display: none;
|
||||
|
|
@ -140,10 +143,6 @@
|
|||
#agentIndicator.speaking {
|
||||
color: #8b4513;
|
||||
}
|
||||
/* Deepen the background while PTT is active */
|
||||
body.ptt-active {
|
||||
background: radial-gradient(ellipse at 50% 44%, #f2caa8 0%, #e8b898 100%);
|
||||
}
|
||||
#controls {
|
||||
position: fixed;
|
||||
top: 12px;
|
||||
|
|
@ -167,20 +166,236 @@
|
|||
transform: translateY(1px);
|
||||
box-shadow: 0 1px 4px rgba(0, 0, 0, 0.15);
|
||||
}
|
||||
|
||||
/* Toast notifications */
|
||||
#toast-container {
|
||||
position: fixed;
|
||||
top: 16px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
width: min(92vw, 480px);
|
||||
max-height: calc(100vh - 32px);
|
||||
overflow-y: auto;
|
||||
overflow-x: hidden;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
z-index: 100;
|
||||
pointer-events: auto;
|
||||
/* Hide scrollbar until hovered */
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: rgba(255,200,140,0.25) transparent;
|
||||
padding-bottom: 4px;
|
||||
}
|
||||
#toast-container::-webkit-scrollbar {
|
||||
width: 4px;
|
||||
}
|
||||
#toast-container::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
#toast-container::-webkit-scrollbar-thumb {
|
||||
background: rgba(255,200,140,0.25);
|
||||
border-radius: 2px;
|
||||
}
|
||||
.toast {
|
||||
pointer-events: auto;
|
||||
background: rgba(28, 22, 16, 0.92);
|
||||
border: 1px solid rgba(255, 200, 140, 0.18);
|
||||
border-radius: 12px;
|
||||
padding: 14px 16px 14px 16px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
box-shadow: 0 4px 24px rgba(0, 0, 0, 0.45);
|
||||
animation: toast-in 0.22s cubic-bezier(0.34, 1.4, 0.64, 1) both;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
max-width: 100%;
|
||||
}
|
||||
.toast.dismissing {
|
||||
animation: toast-out 0.18s ease-in both;
|
||||
}
|
||||
@keyframes toast-in {
|
||||
from { opacity: 0; transform: translateY(-14px) scale(0.96); }
|
||||
to { opacity: 1; transform: translateY(0) scale(1); }
|
||||
}
|
||||
@keyframes toast-out {
|
||||
from { opacity: 1; transform: translateY(0) scale(1); }
|
||||
to { opacity: 0; transform: translateY(-10px) scale(0.96); }
|
||||
}
|
||||
.toast-progress {
|
||||
position: absolute;
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
height: 2px;
|
||||
background: rgba(255, 190, 120, 0.55);
|
||||
width: 100%;
|
||||
transform-origin: left;
|
||||
animation: toast-progress-shrink linear both;
|
||||
}
|
||||
@keyframes toast-progress-shrink {
|
||||
from { transform: scaleX(1); }
|
||||
to { transform: scaleX(0); }
|
||||
}
|
||||
.toast-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-start;
|
||||
gap: 10px;
|
||||
}
|
||||
.toast-title {
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
letter-spacing: 0.07em;
|
||||
color: rgba(255, 200, 140, 0.85);
|
||||
text-transform: uppercase;
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.toast-close {
|
||||
background: none;
|
||||
border: none;
|
||||
color: rgba(255, 245, 235, 0.35);
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
padding: 0 2px;
|
||||
flex-shrink: 0;
|
||||
transition: color 0.15s;
|
||||
}
|
||||
.toast-close:hover {
|
||||
color: rgba(255, 245, 235, 0.85);
|
||||
}
|
||||
.toast-body {
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
line-height: 1.65;
|
||||
color: rgba(255, 245, 235, 0.82);
|
||||
white-space: normal;
|
||||
word-break: break-word;
|
||||
user-select: text;
|
||||
-webkit-user-select: text;
|
||||
}
|
||||
.toast-body p { margin: 0 0 6px; }
|
||||
.toast-body p:last-child { margin-bottom: 0; }
|
||||
.toast-body h1, .toast-body h2, .toast-body h3,
|
||||
.toast-body h4, .toast-body h5, .toast-body h6 {
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
color: rgba(255, 200, 140, 0.95);
|
||||
margin: 8px 0 4px;
|
||||
}
|
||||
.toast-body ul, .toast-body ol {
|
||||
margin: 4px 0 6px;
|
||||
padding-left: 18px;
|
||||
}
|
||||
.toast-body li { margin-bottom: 2px; }
|
||||
.toast-body code {
|
||||
background: rgba(255,255,255,0.07);
|
||||
border-radius: 4px;
|
||||
padding: 1px 5px;
|
||||
font-size: 11px;
|
||||
}
|
||||
.toast-body pre {
|
||||
background: rgba(0,0,0,0.35);
|
||||
border-radius: 6px;
|
||||
padding: 8px 10px;
|
||||
overflow-x: auto;
|
||||
margin: 6px 0;
|
||||
}
|
||||
.toast-body pre code {
|
||||
background: none;
|
||||
padding: 0;
|
||||
font-size: 11px;
|
||||
}
|
||||
.toast-body table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
font-size: 11px;
|
||||
margin: 6px 0;
|
||||
}
|
||||
.toast-body th, .toast-body td {
|
||||
border: 1px solid rgba(255,200,140,0.2);
|
||||
padding: 4px 8px;
|
||||
text-align: left;
|
||||
}
|
||||
.toast-body th {
|
||||
background: rgba(255,200,140,0.08);
|
||||
color: rgba(255,200,140,0.9);
|
||||
font-weight: 600;
|
||||
}
|
||||
.toast-body a {
|
||||
color: rgba(255,200,140,0.85);
|
||||
text-decoration: underline;
|
||||
}
|
||||
.toast-body blockquote {
|
||||
border-left: 3px solid rgba(255,200,140,0.3);
|
||||
margin: 6px 0;
|
||||
padding-left: 10px;
|
||||
color: rgba(255,245,235,0.55);
|
||||
}
|
||||
.toast-body hr {
|
||||
border: none;
|
||||
border-top: 1px solid rgba(255,200,140,0.15);
|
||||
margin: 8px 0;
|
||||
}
|
||||
.toast-choices {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
.toast-choice-btn {
|
||||
background: rgba(255, 200, 140, 0.12);
|
||||
border: 1px solid rgba(255, 200, 140, 0.35);
|
||||
border-radius: 8px;
|
||||
color: rgba(255, 245, 235, 0.90);
|
||||
font-family: "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
font-size: 12px;
|
||||
padding: 6px 14px;
|
||||
cursor: pointer;
|
||||
transition: background 0.15s, border-color 0.15s;
|
||||
flex: 1 1 auto;
|
||||
text-align: center;
|
||||
}
|
||||
.toast-choice-btn:hover {
|
||||
background: rgba(255, 200, 140, 0.25);
|
||||
border-color: rgba(255, 200, 140, 0.65);
|
||||
}
|
||||
.toast-choice-btn:active {
|
||||
background: rgba(255, 200, 140, 0.38);
|
||||
}
|
||||
.toast-choice-btn:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
.toast-image {
|
||||
width: 100%;
|
||||
max-height: 320px;
|
||||
object-fit: contain;
|
||||
border-radius: 8px;
|
||||
display: block;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="controls" data-no-ptt="1">
|
||||
<button id="resetSessionBtn" class="control-btn" type="button" data-no-ptt="1">Reset</button>
|
||||
<div id="controls">
|
||||
<button id="resetSessionBtn" class="control-btn" type="button">Reset</button>
|
||||
</div>
|
||||
<div id="log"><div id="log-inner"></div></div>
|
||||
<div id="agentIndicator">
|
||||
<div id="agentViz"></div>
|
||||
<div id="agentIndicator" data-ptt="1">
|
||||
<div id="agentViz" data-ptt="1"></div>
|
||||
<span class="label"></span>
|
||||
</div>
|
||||
<div id="voiceStatus"></div>
|
||||
<div id="toast-container"></div>
|
||||
<audio id="remoteAudio" autoplay playsinline hidden></audio>
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
|
||||
<script src="/static/three.min.js"></script>
|
||||
<script>
|
||||
const logEl = document.getElementById("log-inner");
|
||||
|
|
@ -190,15 +405,134 @@
|
|||
const agentVizEl = document.getElementById("agentViz");
|
||||
const agentLabel = agentIndicator.querySelector(".label");
|
||||
const resetSessionBtn = document.getElementById("resetSessionBtn");
|
||||
const toastContainer = document.getElementById("toast-container");
|
||||
|
||||
// --- Toast notifications ---
|
||||
const showToast = (kind, content, title, durationMs) => {
|
||||
const toast = document.createElement("div");
|
||||
toast.className = "toast";
|
||||
|
||||
// Header row (title + close button)
|
||||
const header = document.createElement("div");
|
||||
header.className = "toast-header";
|
||||
|
||||
if (title) {
|
||||
const titleEl = document.createElement("span");
|
||||
titleEl.className = "toast-title";
|
||||
titleEl.textContent = title;
|
||||
header.appendChild(titleEl);
|
||||
}
|
||||
|
||||
const closeBtn = document.createElement("button");
|
||||
closeBtn.className = "toast-close";
|
||||
closeBtn.setAttribute("type", "button");
|
||||
closeBtn.setAttribute("aria-label", "Dismiss");
|
||||
closeBtn.textContent = "×";
|
||||
header.appendChild(closeBtn);
|
||||
|
||||
toast.appendChild(header);
|
||||
|
||||
// Body
|
||||
if (kind === "image") {
|
||||
const img = document.createElement("img");
|
||||
img.className = "toast-image";
|
||||
img.src = content;
|
||||
img.alt = title || "image";
|
||||
toast.appendChild(img);
|
||||
} else {
|
||||
const body = document.createElement("div");
|
||||
body.className = "toast-body";
|
||||
// If content looks like HTML, inject directly; otherwise render as markdown.
|
||||
const looksLikeHtml = /^\s*<[a-zA-Z]/.test(content);
|
||||
if (looksLikeHtml) {
|
||||
body.innerHTML = content;
|
||||
} else if (typeof marked !== "undefined") {
|
||||
body.innerHTML = marked.parse(content);
|
||||
} else {
|
||||
body.textContent = content;
|
||||
}
|
||||
toast.appendChild(body);
|
||||
}
|
||||
|
||||
// dismiss must be declared before close button references it
|
||||
const dismiss = () => {
|
||||
toast.classList.add("dismissing");
|
||||
const fallback = setTimeout(() => toast.remove(), 400);
|
||||
toast.addEventListener("animationend", () => { clearTimeout(fallback); toast.remove(); }, { once: true });
|
||||
};
|
||||
|
||||
closeBtn.addEventListener("click", (e) => { e.stopPropagation(); dismiss(); });
|
||||
toastContainer.prepend(toast);
|
||||
toastContainer.scrollTop = 0;
|
||||
};
|
||||
|
||||
// --- Choice toasts (ask_user tool) ---
|
||||
const showChoice = (requestId, question, choices, title) => {
|
||||
const toast = document.createElement("div");
|
||||
toast.className = "toast";
|
||||
|
||||
// Header
|
||||
const header = document.createElement("div");
|
||||
header.className = "toast-header";
|
||||
if (title) {
|
||||
const titleEl = document.createElement("span");
|
||||
titleEl.className = "toast-title";
|
||||
titleEl.textContent = title;
|
||||
header.appendChild(titleEl);
|
||||
}
|
||||
const closeBtn = document.createElement("button");
|
||||
closeBtn.className = "toast-close";
|
||||
closeBtn.setAttribute("type", "button");
|
||||
closeBtn.setAttribute("aria-label", "Dismiss");
|
||||
closeBtn.textContent = "×";
|
||||
header.appendChild(closeBtn);
|
||||
toast.appendChild(header);
|
||||
|
||||
// Question body
|
||||
const body = document.createElement("div");
|
||||
body.className = "toast-body";
|
||||
body.textContent = question;
|
||||
toast.appendChild(body);
|
||||
|
||||
// Choice buttons
|
||||
const choicesEl = document.createElement("div");
|
||||
choicesEl.className = "toast-choices";
|
||||
|
||||
const dismiss = () => {
|
||||
toast.classList.add("dismissing");
|
||||
const fallback = setTimeout(() => toast.remove(), 400);
|
||||
toast.addEventListener("animationend", () => { clearTimeout(fallback); toast.remove(); }, { once: true });
|
||||
};
|
||||
|
||||
choices.forEach((label) => {
|
||||
const btn = document.createElement("button");
|
||||
btn.className = "toast-choice-btn";
|
||||
btn.setAttribute("type", "button");
|
||||
btn.textContent = label;
|
||||
btn.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
// Disable all buttons to prevent double-send
|
||||
choicesEl.querySelectorAll(".toast-choice-btn").forEach((b) => { b.disabled = true; });
|
||||
sendJson({ type: "ui-response", request_id: requestId, value: label });
|
||||
dismiss();
|
||||
});
|
||||
choicesEl.appendChild(btn);
|
||||
});
|
||||
toast.appendChild(choicesEl);
|
||||
|
||||
closeBtn.addEventListener("click", (e) => { e.stopPropagation(); dismiss(); });
|
||||
toastContainer.prepend(toast);
|
||||
toastContainer.scrollTop = 0;
|
||||
};
|
||||
|
||||
// --- Agent state indicator ---
|
||||
const STATES = { idle: "idle", listening: "listening", thinking: "thinking", speaking: "speaking" };
|
||||
const STATE_COLORS = {
|
||||
[STATES.idle]: 0xfff5eb,
|
||||
[STATES.listening]: 0xfff5eb,
|
||||
[STATES.thinking]: 0xfff5eb,
|
||||
[STATES.speaking]: 0xfff5eb,
|
||||
};
|
||||
const STATE_COLORS = {
|
||||
[STATES.idle]: 0xfff5eb,
|
||||
[STATES.listening]: 0xfff5eb,
|
||||
[STATES.thinking]: 0xfff5eb,
|
||||
[STATES.speaking]: 0xfff5eb,
|
||||
};
|
||||
let agentState = STATES.idle;
|
||||
let agentVisualizer = null;
|
||||
let lastRemoteAudioActivityS = 0;
|
||||
|
|
@ -248,7 +582,8 @@
|
|||
powerPreference: "high-performance",
|
||||
});
|
||||
renderer.setPixelRatio(1);
|
||||
renderer.setClearColor(0xa09b96, 1);
|
||||
renderer.setClearColor(0xe8e4e0, 1);
|
||||
renderer.domElement.dataset.ptt = "1";
|
||||
agentVizEl.innerHTML = "";
|
||||
agentVizEl.appendChild(renderer.domElement);
|
||||
|
||||
|
|
@ -358,12 +693,12 @@
|
|||
let deformScale = 1.0;
|
||||
let ringScale = 1.0; // uniform xz scale — used for thickness throb when thinking
|
||||
let spinSpeed = 0.0;
|
||||
// Card background colour lerp: 0 = idle coral, 1 = dark listening
|
||||
// Card background colour lerp: 0 = idle coral, 1 = dark coral (PTT/listening)
|
||||
let cardColorT = 0.0;
|
||||
let connectedT = 0.0; // 0 = gray (disconnected), 1 = coral (connected)
|
||||
const CARD_GRAY_RGB = [160, 155, 150]; // disconnected gray
|
||||
const CARD_IDLE_RGB = [212, 85, 63]; // #d4553f
|
||||
const CARD_LISTEN_RGB = [120, 40, 28]; // dark desaturated coral
|
||||
const CARD_GRAY_RGB = [232, 228, 224]; // #e8e4e0 — disconnected light warm gray
|
||||
const CARD_IDLE_RGB = [212, 85, 63]; // #d4553f — connected idle coral
|
||||
const CARD_LISTEN_RGB = [120, 40, 28]; // #782c1c — PTT active dark coral
|
||||
|
||||
const setStateColor = (_state) => { /* no-op: MeshBasicMaterial, colour is fixed */ };
|
||||
|
||||
|
|
@ -696,7 +1031,6 @@
|
|||
|
||||
const setPushToTalkState = (pressed, notifyServer = true) => {
|
||||
pttPressed = pressed;
|
||||
document.body.classList.toggle("ptt-active", pressed);
|
||||
setMicCaptureEnabled(pressed);
|
||||
if (notifyServer && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ type: "voice-ptt", pressed }));
|
||||
|
|
@ -947,26 +1281,27 @@
|
|||
if (!appStarted) {
|
||||
await bootstrap();
|
||||
}
|
||||
if (sendUserMessage("/reset")) {
|
||||
showStatus("Reset command sent.", 1500);
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
sendJson({ type: "command", command: "reset" });
|
||||
showStatus("Session reset.", 1500);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// --- Whole-screen PTT pointer handling ---
|
||||
// --- Center-card PTT pointer handling ---
|
||||
// Only touches that land on #agentIndicator / #agentViz (data-ptt="1") trigger PTT.
|
||||
// We track active pointer IDs so multi-touch doesn't double-fire.
|
||||
const activePointers = new Set();
|
||||
|
||||
document.addEventListener("pointerdown", async (event) => {
|
||||
if (event.target instanceof Element && event.target.closest("[data-no-ptt='1']")) {
|
||||
if (!(event.target instanceof Element) || !event.target.closest("[data-ptt='1']")) {
|
||||
return;
|
||||
}
|
||||
activePointers.add(event.pointerId);
|
||||
if (!appStarted) {
|
||||
await bootstrap();
|
||||
return;
|
||||
}
|
||||
ensureVisualizerAudioMeter();
|
||||
activePointers.add(event.pointerId);
|
||||
if (activePointers.size === 1) beginPushToTalk();
|
||||
}, { passive: false });
|
||||
|
||||
|
|
@ -1020,6 +1355,30 @@
|
|||
if (agentState !== STATES.listening && STATES[newState]) {
|
||||
setAgentState(newState);
|
||||
}
|
||||
} else if (msg.role === "toast") {
|
||||
try {
|
||||
const t = JSON.parse(msg.text || "{}");
|
||||
showToast(
|
||||
t.kind || "text",
|
||||
t.content || "",
|
||||
t.title || "",
|
||||
typeof t.duration_ms === "number" ? t.duration_ms : 6000,
|
||||
);
|
||||
} catch (_) {
|
||||
showToast("text", msg.text || "", "", 6000);
|
||||
}
|
||||
} else if (msg.role === "choice") {
|
||||
try {
|
||||
const c = JSON.parse(msg.text || "{}");
|
||||
showChoice(
|
||||
c.request_id || "",
|
||||
c.question || "",
|
||||
Array.isArray(c.choices) ? c.choices : [],
|
||||
c.title || "",
|
||||
);
|
||||
} catch (_) {
|
||||
// Malformed choice payload — ignore.
|
||||
}
|
||||
} else if (msg.role === "wisper") {
|
||||
// suppress wisper debug output
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue