From db4ce8b14fcf006f03497649c11c3e2434fb5281 Mon Sep 17 00:00:00 2001 From: kacper Date: Thu, 12 Mar 2026 09:25:15 -0400 Subject: [PATCH] stable --- .gitignore | 28 + AGENTS.md | 40 + app.py | 774 ++++++++++++++++- frontend/biome.json | 3 + frontend/bun.lock | 117 +++ frontend/package.json | 1 + frontend/src/App.tsx | 284 ++++++- frontend/src/audioMeter.ts | 2 +- frontend/src/components/AgentIndicator.tsx | 8 +- frontend/src/components/CardFeed.tsx | 353 ++++++++ frontend/src/components/Controls.tsx | 95 ++- frontend/src/components/FAB.tsx | 45 + frontend/src/components/LogPanel.tsx | 211 ++++- frontend/src/components/TextInput.tsx | 188 +++++ frontend/src/components/Toast.tsx | 109 --- frontend/src/hooks/usePTT.ts | 57 +- frontend/src/hooks/usePushToTalk.ts | 54 -- frontend/src/hooks/useWebRTC.ts | 705 ++++++++++------ frontend/src/index.css | 934 +++++++++++++++++---- frontend/src/types.ts | 81 +- supertonic_gateway.py | 233 +++-- voice_rtc.py | 58 +- 22 files changed, 3557 insertions(+), 823 deletions(-) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 frontend/src/components/CardFeed.tsx create mode 100644 frontend/src/components/FAB.tsx create mode 100644 frontend/src/components/TextInput.tsx delete mode 100644 frontend/src/components/Toast.tsx delete mode 100644 frontend/src/hooks/usePushToTalk.ts diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..990e322 --- /dev/null +++ b/.gitignore @@ -0,0 +1,28 @@ +# Local secrets and certs +.certs/ +.env +.env.* +!.env.example + +# Python runtime +.venv/ +venv/ +__pycache__/ +*.pyc +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ + +# Local app data +cards.db +.todos/ + +# Frontend build and dependencies +frontend/node_modules/ +frontend/dist/ +frontend/.vite/ + +# Editor / OS noise +.DS_Store diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4e8386d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,40 @@ +# Agent Instructions + +## Project layout + +- `app.py` — FastAPI server (writable) +- `frontend/` — Preact + TypeScript UI built with Vite +- `voice_rtc.py`, `supertonic_gateway.py`, `wisper.py` — read-only; do not modify +- `start.sh` — read-only startup script +- `.env.voice` — local env overrides (writable); sourced before `start.sh` defaults + +## Toolchain + +All frontend commands run from `frontend/` using `~/.bun/bin/bun`. + +| Task | Command | +|---|---| +| Lint + format check | `~/.bun/bin/bun run check` | +| Build | `~/.bun/bin/bun run build` | +| Dead code | `~/.bun/bin/bunx knip` | +| Format a file | `~/.bun/bin/bun run biome format --write ` | + +Always run `check` and `build` after making frontend changes. Both must pass with no errors before finishing. + +## Linting rules + +- Linter: Biome (config at `frontend/biome.json`) +- No `biome-ignore` suppressions — fix the code or disable the rule in `biome.json` +- All font sizes in CSS must use `rem`, not `px` + +## Dead code + +Run `~/.bun/bin/bunx knip` from `frontend/` to find unused files, exports, and types. The only expected false positive is `dist/assets/` build output. + +## Backend + +- Card instances are file-backed in `NANOBOT_WORKSPACE/cards/instances//card.json` +- Card HTML snapshots are stored beside metadata in `NANOBOT_WORKSPACE/cards/instances//render.html` +- Templates live in `NANOBOT_WORKSPACE/cards/templates//template.html` with `manifest.json` +- `GET /cards` returns all non-archived cards ordered by lane, state, priority, then update time +- `DELETE /cards/{id}` removes a card on dismiss diff --git a/app.py b/app.py index 2d48661..348f2e6 100644 --- a/app.py +++ b/app.py @@ -1,12 +1,20 @@ import asyncio import contextlib +import html import json +import os +import re +import shutil +import sys +from datetime import datetime, timezone from pathlib import Path -from typing import Any, Awaitable, Callable +from typing import Any +from urllib.parse import urlparse, urlunparse +import httpx from fastapi import FastAPI, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, JSONResponse +from fastapi.responses import FileResponse, JSONResponse, Response from fastapi.staticfiles import StaticFiles from supertonic_gateway import SuperTonicGateway @@ -15,6 +23,29 @@ from voice_rtc import WebRTCVoiceSession BASE_DIR = Path(__file__).resolve().parent DIST_DIR = BASE_DIR / "frontend" / "dist" +NANOBOT_CONFIG_PATH = Path( + os.getenv("NANOBOT_CONFIG_PATH", str(Path.home() / ".nanobot" / "config.json")) +).expanduser() +NANOBOT_WORKSPACE = Path( + os.getenv("NANOBOT_WORKSPACE", str(Path.home() / ".nanobot")) +).expanduser() +NANOBOT_SCRIPT_WORKSPACE = Path( + os.getenv("NANOBOT_SCRIPT_WORKSPACE", str(NANOBOT_WORKSPACE / "workspace")) +).expanduser() +CARDS_ROOT = NANOBOT_WORKSPACE / "cards" +CARD_INSTANCES_DIR = CARDS_ROOT / "instances" +CARD_TEMPLATES_DIR = CARDS_ROOT / "templates" +TEMPLATES_CONTEXT_PATH = NANOBOT_WORKSPACE / "CARD_TEMPLATES.md" +MAX_TEMPLATES_IN_PROMPT = 12 +MAX_TEMPLATE_HTML_CHARS = 4000 +_INVALID_TEMPLATE_KEY_CHARS = re.compile(r"[^a-z0-9_-]+") +_CARD_ID_PATTERN = re.compile(r"^[a-zA-Z0-9_-]{1,128}$") +_CARD_LANE_ORDER = {"attention": 0, "work": 1, "context": 2, "history": 3} +_CARD_STATE_ORDER = {"active": 0, "stale": 1, "resolved": 2, "superseded": 3, "archived": 4} +_MAX_SCRIPT_PROXY_ARGS = 16 +_MAX_SCRIPT_PROXY_STDERR_CHARS = 2000 +CARD_INSTANCES_DIR.mkdir(parents=True, exist_ok=True) +CARD_TEMPLATES_DIR.mkdir(parents=True, exist_ok=True) app = FastAPI(title="Nanobot SuperTonic Wisper Web") @@ -27,25 +58,719 @@ app.add_middleware( gateway = SuperTonicGateway() -# Session store: one voice session per connection (keyed by a simple counter). -# For this single-user app we keep at most one active session at a time. _active_session: WebRTCVoiceSession | None = None _active_queue: asyncio.Queue | None = None _sender_task: asyncio.Task | None = None +# --------------------------------------------------------------------------- +# Cards (file-backed) +# --------------------------------------------------------------------------- + + +def _normalize_template_key(raw: str) -> str: + key = _INVALID_TEMPLATE_KEY_CHARS.sub("-", raw.strip().lower()).strip("-") + return key[:64] + + +def _normalize_card_id(raw: str) -> str: + card_id = raw.strip() + return card_id if _CARD_ID_PATTERN.fullmatch(card_id) else "" + + +def _card_instance_dir(card_id: str) -> Path | None: + card_id_clean = _normalize_card_id(card_id) + if not card_id_clean: + return None + return CARD_INSTANCES_DIR / card_id_clean + + +def _card_meta_path(card_id: str) -> Path | None: + instance_dir = _card_instance_dir(card_id) + if instance_dir is None: + return None + return instance_dir / "card.json" + + +def _card_state_path(card_id: str) -> Path | None: + instance_dir = _card_instance_dir(card_id) + if instance_dir is None: + return None + return instance_dir / "state.json" + + +def _decode_object(raw: str) -> dict[str, Any] | None: + try: + payload = json.loads(raw) + except (TypeError, ValueError): + return None + return payload if isinstance(payload, dict) else None + + +def _coerce_card_record(raw: dict[str, Any]) -> dict[str, Any] | None: + card_id = _normalize_card_id(str(raw.get("id", ""))) + if not card_id: + return None + + kind = str(raw.get("kind", "text") or "text").strip().lower() + if kind not in {"text", "question"}: + kind = "text" + + lane = str(raw.get("lane", "context") or "context").strip().lower() + if lane not in _CARD_LANE_ORDER: + lane = "context" + + state = str(raw.get("state", "active") or "active").strip().lower() + if state not in _CARD_STATE_ORDER: + state = "active" + + try: + priority = int(raw.get("priority", 50)) + except (TypeError, ValueError): + priority = 50 + priority = max(0, min(priority, 100)) + + raw_choices = raw.get("choices", []) + choices = [str(choice) for choice in raw_choices] if isinstance(raw_choices, list) else [] + raw_template_state = raw.get("template_state", {}) + template_state = raw_template_state if isinstance(raw_template_state, dict) else {} + + return { + "id": card_id, + "kind": kind, + "title": str(raw.get("title", "")), + "content": str(raw.get("content", "")) if kind == "question" else "", + "question": str(raw.get("question", "")), + "choices": choices, + "response_value": str(raw.get("response_value", "")), + "slot": str(raw.get("slot", "")), + "lane": lane, + "priority": priority, + "state": state, + "template_key": str(raw.get("template_key", "")), + "template_state": template_state, + "context_summary": str(raw.get("context_summary", "")), + "chat_id": str(raw.get("chat_id", "web") or "web"), + "created_at": str(raw.get("created_at", "")), + "updated_at": str(raw.get("updated_at", "")), + } + + +def _json_script_text(payload: dict[str, Any]) -> str: + return json.dumps(payload, ensure_ascii=False).replace(" str: + if card.get("kind") != "text": + return str(card.get("content", "")) + + template_key = str(card.get("template_key", "")).strip() + if not template_key: + return "" + + html_path = _template_html_path(template_key) + try: + template_html = html_path.read_text(encoding="utf-8") + except Exception: + return ( + "
" + f"Missing template: {html.escape(template_key)}" + "
" + ) + + state_payload = card.get("template_state", {}) + if not isinstance(state_payload, dict): + state_payload = {} + + card_id = html.escape(str(card.get("id", ""))) + safe_template_key = html.escape(template_key) + return ( + f'
' + f'' + f"{template_html}" + "
" + ) + + +def _load_card(card_id: str) -> dict[str, Any] | None: + meta_path = _card_meta_path(card_id) + if meta_path is None or not meta_path.exists(): + return None + try: + raw = json.loads(meta_path.read_text(encoding="utf-8")) + except Exception: + return None + if not isinstance(raw, dict): + return None + + state_path = _card_state_path(card_id) + if state_path is not None and state_path.exists(): + try: + raw_state = json.loads(state_path.read_text(encoding="utf-8")) + except Exception: + raw_state = {} + if isinstance(raw_state, dict): + raw["template_state"] = raw_state + + card = _coerce_card_record(raw) + if card is None: + return None + + card["content"] = _materialize_card_content(card) + return card + + +def _write_card(card: dict[str, Any]) -> dict[str, Any] | None: + normalized = _coerce_card_record(card) + if normalized is None: + return None + + now = datetime.now(timezone.utc).isoformat() + existing = _load_card(normalized["id"]) + if existing is not None: + normalized["created_at"] = existing.get("created_at") or normalized.get("created_at") or now + else: + normalized["created_at"] = normalized.get("created_at") or now + normalized["updated_at"] = normalized.get("updated_at") or now + + instance_dir = _card_instance_dir(normalized["id"]) + meta_path = _card_meta_path(normalized["id"]) + state_path = _card_state_path(normalized["id"]) + if instance_dir is None or meta_path is None or state_path is None: + return None + + instance_dir.mkdir(parents=True, exist_ok=True) + + template_state = normalized.pop("template_state", {}) + meta_path.write_text( + json.dumps(normalized, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + if normalized["kind"] == "text": + state_path.write_text( + json.dumps(template_state if isinstance(template_state, dict) else {}, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + else: + state_path.unlink(missing_ok=True) + + return _load_card(normalized["id"]) + + +def _sort_cards(cards: list[dict[str, Any]]) -> list[dict[str, Any]]: + return sorted( + cards, + key=lambda item: ( + _CARD_LANE_ORDER.get(str(item.get("lane", "context")), 99), + _CARD_STATE_ORDER.get(str(item.get("state", "active")), 99), + -int(item.get("priority", 0) or 0), + str(item.get("updated_at", "")), + str(item.get("created_at", "")), + ), + reverse=False, + ) + + +def _load_cards() -> list[dict[str, Any]]: + CARD_INSTANCES_DIR.mkdir(parents=True, exist_ok=True) + cards: list[dict[str, Any]] = [] + for instance_dir in CARD_INSTANCES_DIR.iterdir(): + if not instance_dir.is_dir(): + continue + card = _load_card(instance_dir.name) + if card is None: + continue + if card.get("state") == "archived": + continue + cards.append(card) + return _sort_cards(cards) + + +def _find_card_by_slot(slot: str, *, chat_id: str) -> dict[str, Any] | None: + target_slot = slot.strip() + if not target_slot: + return None + for card in _load_cards(): + if str(card.get("chat_id", "")).strip() != chat_id: + continue + if str(card.get("slot", "")).strip() == target_slot: + return card + return None + + +def _persist_card(card: dict[str, Any]) -> dict[str, Any] | None: + normalized = _coerce_card_record(card) + if normalized is None: + return None + + existing_same_slot = None + slot = str(normalized.get("slot", "")).strip() + chat_id = str(normalized.get("chat_id", "web") or "web") + if slot: + existing_same_slot = _find_card_by_slot(slot, chat_id=chat_id) + + if existing_same_slot is not None and existing_same_slot["id"] != normalized["id"]: + superseded = dict(existing_same_slot) + superseded["state"] = "superseded" + superseded["updated_at"] = datetime.now(timezone.utc).isoformat() + _write_card(superseded) + + return _write_card(normalized) + + +def _delete_card(card_id: str) -> bool: + instance_dir = _card_instance_dir(card_id) + if instance_dir is None or not instance_dir.exists(): + return False + shutil.rmtree(instance_dir, ignore_errors=True) + return True + + +def _template_dir(template_key: str) -> Path: + return CARD_TEMPLATES_DIR / template_key + + +def _template_html_path(template_key: str) -> Path: + return _template_dir(template_key) / "template.html" + + +def _template_meta_path(template_key: str) -> Path: + return _template_dir(template_key) / "manifest.json" + + +def _read_template_meta(template_key: str) -> dict[str, Any]: + meta_path = _template_meta_path(template_key) + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + return meta if isinstance(meta, dict) else {} + except Exception: + return {} + + +def _list_templates(limit: int | None = None) -> list[dict[str, Any]]: + CARD_TEMPLATES_DIR.mkdir(parents=True, exist_ok=True) + templates: list[dict[str, Any]] = [] + + for template_dir in CARD_TEMPLATES_DIR.iterdir(): + if not template_dir.is_dir(): + continue + key = _normalize_template_key(template_dir.name) + if not key: + continue + + html_path = _template_html_path(key) + if not html_path.exists(): + continue + try: + content = html_path.read_text(encoding="utf-8") + except Exception: + continue + + stat = html_path.stat() + meta = _read_template_meta(key) + if bool(meta.get("deprecated")): + continue + created_at = str(meta.get("created_at") or datetime.fromtimestamp(stat.st_ctime, timezone.utc).isoformat()) + updated_at = str(meta.get("updated_at") or datetime.fromtimestamp(stat.st_mtime, timezone.utc).isoformat()) + templates.append( + { + "id": key, + "key": key, + "title": str(meta.get("title", "")), + "content": content, + "notes": str(meta.get("notes", "")), + "example_state": meta.get("example_state", {}), + "created_at": created_at, + "updated_at": updated_at, + "file_url": f"/card-templates/{key}/template.html", + } + ) + + templates.sort(key=lambda item: item["updated_at"], reverse=True) + if limit is not None: + return templates[: max(0, limit)] + return templates + + +def _render_templates_markdown(rows: list[dict[str, Any]]) -> str: + lines = [ + "# Card Templates", + "", + "These are user-approved template layouts for `mcp_display_render_card` cards.", + "Each card instance should provide a `template_key` and a `template_state` JSON object.", + "Use a matching template when the request intent fits.", + "Do not rewrite the HTML layout when an existing template already fits; fill the template_state instead.", + "", + ] + for row in rows: + key = str(row.get("key", "")).strip() or "unnamed" + title = str(row.get("title", "")).strip() or "(untitled)" + notes = str(row.get("notes", "")).strip() or "(no usage notes)" + content = str(row.get("content", "")).strip() + example_state = row.get("example_state", {}) + if len(content) > MAX_TEMPLATE_HTML_CHARS: + content = content[:MAX_TEMPLATE_HTML_CHARS] + "\n" + html_lines = [f" {line}" for line in content.splitlines()] if content else [" "] + state_text = json.dumps(example_state, indent=2, ensure_ascii=False) if isinstance(example_state, dict) else "{}" + state_lines = [f" {line}" for line in state_text.splitlines()] + lines.extend( + [ + f"## {key}", + f"- Title: {title}", + f"- Usage: {notes}", + "- Example State:", + *state_lines, + "- HTML:", + *html_lines, + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def _sync_templates_context_file() -> None: + try: + rows = _list_templates(limit=MAX_TEMPLATES_IN_PROMPT) + if not rows: + TEMPLATES_CONTEXT_PATH.unlink(missing_ok=True) + return + TEMPLATES_CONTEXT_PATH.parent.mkdir(parents=True, exist_ok=True) + TEMPLATES_CONTEXT_PATH.write_text(_render_templates_markdown(rows), encoding="utf-8") + except Exception: + return + + +def _to_typed_message(event_dict: dict[str, Any]) -> dict[str, Any] | None: + role = str(event_dict.get("role", "")).strip() + text = str(event_dict.get("text", "")) + timestamp = str(event_dict.get("timestamp", "")) + + if role == "agent-state": + return {"type": "agent_state", "state": text} + + if role in {"nanobot", "nanobot-progress", "nanobot-tool", "system", "user"}: + return { + "type": "message", + "role": role, + "content": text, + "is_progress": role in {"nanobot-progress", "nanobot-tool"}, + "is_tool_hint": role == "nanobot-tool", + "timestamp": timestamp, + } + + if role == "card": + payload = _decode_object(text) + if payload is None: + return None + card = _coerce_card_record(payload) + if card is None: + return None + card["type"] = "card" + return card + + return None + + +# --------------------------------------------------------------------------- +# Nanobot config / HA proxy +# --------------------------------------------------------------------------- + + +def _load_nanobot_config() -> dict[str, Any]: + try: + return json.loads(NANOBOT_CONFIG_PATH.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _get_home_assistant_mcp_config() -> tuple[str, dict[str, str]]: + cfg = _load_nanobot_config() + tools = cfg.get("tools") if isinstance(cfg, dict) else {} + if not isinstance(tools, dict): + raise RuntimeError("nanobot config missing tools section") + + mcp_servers = tools.get("mcpServers") + if not isinstance(mcp_servers, dict): + raise RuntimeError("nanobot config missing tools.mcpServers section") + + raw_server = mcp_servers.get("home assistant") or mcp_servers.get("home_assistant") + if not isinstance(raw_server, dict): + raise RuntimeError("home assistant MCP server is not configured") + + url = str(raw_server.get("url", "")).strip() + if not url: + raise RuntimeError("home assistant MCP server URL is empty") + + raw_headers = raw_server.get("headers", {}) + headers: dict[str, str] = {} + if isinstance(raw_headers, dict): + for k, v in raw_headers.items(): + headers[str(k)] = str(v) + + return url, headers + + +def _home_assistant_origin(mcp_url: str) -> str: + parsed = urlparse(mcp_url.strip()) + return urlunparse(parsed._replace(path="", params="", query="", fragment="")).rstrip("/") + + +def _normalize_home_assistant_proxy_path(target_path: str) -> str: + normalized = "/" + target_path.lstrip("/") + if normalized == "/": + raise ValueError("target path is required") + if normalized == "/api" or normalized.startswith("/api/"): + return normalized + return f"/api{normalized}" + + +def _resolve_workspace_script(script_path: str) -> Path: + normalized = script_path.strip().lstrip("/") + if not normalized: + raise ValueError("script path is required") + + root = NANOBOT_SCRIPT_WORKSPACE.resolve() + candidate = (root / normalized).resolve() + try: + candidate.relative_to(root) + except ValueError as exc: + raise ValueError("script path escapes workspace") from exc + + if not candidate.is_file(): + raise ValueError(f"script not found: {normalized}") + if candidate.suffix.lower() != ".py": + raise ValueError("only Python scripts are supported") + return candidate + + +def _script_proxy_args(request: Request) -> list[str]: + unknown_keys = sorted({key for key in request.query_params.keys() if key != "arg"}) + if unknown_keys: + raise ValueError( + "unsupported script query parameters: " + ", ".join(unknown_keys) + ) + + args = [str(value) for value in request.query_params.getlist("arg")] + if len(args) > _MAX_SCRIPT_PROXY_ARGS: + raise ValueError( + f"too many script arguments ({len(args)} > {_MAX_SCRIPT_PROXY_ARGS})" + ) + return args + + +# --------------------------------------------------------------------------- +# API routes +# --------------------------------------------------------------------------- + + @app.get("/health") async def health() -> JSONResponse: return JSONResponse({"status": "ok"}) +@app.api_route( + "/ha/proxy/{target_path:path}", + methods=["GET", "POST", "PUT", "PATCH", "DELETE"], +) +async def home_assistant_proxy(target_path: str, request: Request) -> Response: + raw_target = target_path.strip() + if not raw_target: + return JSONResponse({"error": "target path is required"}, status_code=400) + + try: + mcp_url, auth_headers = _get_home_assistant_mcp_config() + origin = _home_assistant_origin(mcp_url) + api_path = _normalize_home_assistant_proxy_path(raw_target) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + except RuntimeError as exc: + return JSONResponse({"error": str(exc)}, status_code=502) + + target_url = f"{origin}{api_path}" + if request.url.query: + target_url = f"{target_url}?{request.url.query}" + + outbound_headers = dict(auth_headers) + incoming_content_type = request.headers.get("content-type") + if incoming_content_type: + outbound_headers["Content-Type"] = incoming_content_type + incoming_accept = request.headers.get("accept") + if incoming_accept: + outbound_headers["Accept"] = incoming_accept + + outbound_body = None if request.method in {"GET", "HEAD"} else await request.body() + try: + async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client: + upstream = await client.request( + request.method, + target_url, + headers=outbound_headers, + content=outbound_body, + ) + except httpx.RequestError as exc: + return JSONResponse( + {"error": f"Home Assistant connection failed: {exc}"}, + status_code=502, + ) + + media_type = upstream.headers.get("content-type") or "application/json" + return Response( + content=upstream.content, + status_code=upstream.status_code, + media_type=media_type, + ) + + +@app.get("/script/proxy/{script_path:path}") +async def workspace_script_proxy(script_path: str, request: Request) -> JSONResponse: + try: + script_file = _resolve_workspace_script(script_path) + args = _script_proxy_args(request) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + + try: + process = await asyncio.create_subprocess_exec( + sys.executable, + str(script_file), + *args, + cwd=str(script_file.parent), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=60.0) + except asyncio.TimeoutError: + with contextlib.suppress(ProcessLookupError): + process.kill() + return JSONResponse({"error": "script execution timed out"}, status_code=504) + except OSError as exc: + return JSONResponse({"error": f"failed to start script: {exc}"}, status_code=502) + + stderr_text = stderr.decode("utf-8", errors="replace").strip() + if process.returncode != 0: + return JSONResponse( + { + "error": f"script exited with code {process.returncode}", + "stderr": stderr_text[:_MAX_SCRIPT_PROXY_STDERR_CHARS], + }, + status_code=502, + ) + + stdout_text = stdout.decode("utf-8", errors="replace").strip() + try: + payload = json.loads(stdout_text) + except json.JSONDecodeError as exc: + return JSONResponse( + { + "error": f"script did not return valid JSON: {exc}", + "stderr": stderr_text[:_MAX_SCRIPT_PROXY_STDERR_CHARS], + }, + status_code=502, + ) + + return JSONResponse(payload) + + +@app.get("/cards") +async def get_cards() -> JSONResponse: + return JSONResponse(_load_cards()) + + +@app.delete("/cards/{card_id}") +async def delete_card(card_id: str) -> JSONResponse: + if not _normalize_card_id(card_id): + return JSONResponse({"error": "invalid card id"}, status_code=400) + _delete_card(card_id) + return JSONResponse({"status": "ok"}) + + +@app.get("/templates") +async def get_templates() -> JSONResponse: + return JSONResponse(_list_templates()) + + +@app.post("/templates") +async def save_template(request: Request) -> JSONResponse: + payload = await request.json() + key = _normalize_template_key(str(payload.get("key", ""))) + title = str(payload.get("title", "")).strip() + content = str(payload.get("content", "")).strip() + notes = str(payload.get("notes", "")).strip() + example_state = payload.get("example_state", {}) + if not isinstance(example_state, dict): + example_state = {} + + if not key: + return JSONResponse({"error": "template key is required"}, status_code=400) + if not content: + return JSONResponse({"error": "template content is required"}, status_code=400) + + template_dir = _template_dir(key) + template_dir.mkdir(parents=True, exist_ok=True) + now = datetime.now(timezone.utc).isoformat() + existing_meta = _read_template_meta(key) + created_at = str(existing_meta.get("created_at") or now) + + _template_html_path(key).write_text(content, encoding="utf-8") + _template_meta_path(key).write_text( + json.dumps( + { + "key": key, + "title": title, + "notes": notes, + "example_state": example_state, + "created_at": created_at, + "updated_at": now, + }, + indent=2, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + + _sync_templates_context_file() + return JSONResponse( + { + "status": "ok", + "id": key, + "key": key, + "example_state": example_state, + "file_url": f"/card-templates/{key}/template.html", + } + ) + + +@app.delete("/templates/{template_key}") +async def delete_template(template_key: str) -> JSONResponse: + key = _normalize_template_key(template_key) + if not key: + return JSONResponse({"error": "invalid template key"}, status_code=400) + shutil.rmtree(_template_dir(key), ignore_errors=True) + _sync_templates_context_file() + return JSONResponse({"status": "ok", "key": key}) + + +@app.post("/message") +async def post_message(request: Request) -> JSONResponse: + payload = await request.json() + text = str(payload.get("text", "")).strip() + metadata = payload.get("metadata", {}) + if not text: + return JSONResponse({"error": "empty message"}, status_code=400) + if not isinstance(metadata, dict): + metadata = {} + await gateway.send_user_message(text, metadata=metadata) + return JSONResponse({"status": "ok"}) + + @app.post("/rtc/offer") async def rtc_offer(request: Request) -> JSONResponse: global _active_session, _active_queue, _sender_task payload = await request.json() - # Tear down any previous session cleanly. if _active_session is not None: await _active_session.close() _active_session = None @@ -76,9 +801,7 @@ async def rtc_offer(request: Request) -> JSONResponse: status_code=503, ) - # Connect to nanobot if not already connected. - await gateway.spawn_tui() - + await gateway.connect_nanobot() return JSONResponse(answer) @@ -105,20 +828,45 @@ async def _sender_loop( if event.role == "nanobot-tts": await voice_session.queue_output_text(event.text) continue - voice_session.send_to_datachannel(event.to_dict()) + typed_event = _to_typed_message(event.to_dict()) + if typed_event is None: + continue + if typed_event.get("type") == "card": + persisted = _persist_card(typed_event) + if persisted is None: + continue + payload = dict(persisted) + payload["type"] = "card" + voice_session.send_to_datachannel(payload) + continue + voice_session.send_to_datachannel(typed_event) + + +@app.on_event("startup") +async def on_startup() -> None: + CARD_INSTANCES_DIR.mkdir(parents=True, exist_ok=True) + CARD_TEMPLATES_DIR.mkdir(parents=True, exist_ok=True) + _sync_templates_context_file() -# Serve the Vite-built frontend as static files. -# This must come AFTER all API routes so the API endpoints are not shadowed. if DIST_DIR.exists(): - # Mount assets sub-directory (hashed JS/CSS) assets_dir = DIST_DIR / "assets" if assets_dir.exists(): app.mount("/assets", StaticFiles(directory=str(assets_dir)), name="assets") + if CARD_TEMPLATES_DIR.exists(): + app.mount( + "/card-templates", + StaticFiles(directory=str(CARD_TEMPLATES_DIR)), + name="card-templates", + ) @app.get("/{full_path:path}") async def spa_fallback(full_path: str) -> FileResponse: candidate = DIST_DIR / full_path if candidate.is_file(): return FileResponse(str(candidate)) - return FileResponse(str(DIST_DIR / "index.html")) + response = FileResponse(str(DIST_DIR / "index.html")) + response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate" + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + return response diff --git a/frontend/biome.json b/frontend/biome.json index 78c4922..09c1147 100644 --- a/frontend/biome.json +++ b/frontend/biome.json @@ -18,6 +18,9 @@ "enabled": true, "rules": { "recommended": true, + "a11y": { + "useGenericFontNames": "off" + }, "correctness": { "noUnusedVariables": "error", "noUnusedImports": "error", diff --git a/frontend/bun.lock b/frontend/bun.lock index a8401b1..498a5d1 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -13,6 +13,7 @@ "@biomejs/biome": "^2.4.6", "@preact/preset-vite": "^2.8.1", "@types/three": "^0.165.0", + "knip": "^5.86.0", "typescript": "^5.4.5", "vite": "^5.2.11", }, @@ -79,6 +80,12 @@ "@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.4.6", "", { "os": "win32", "cpu": "x64" }, "sha512-7++XhnsPlr1HDbor5amovPjOH6vsrFOCdp93iKXhFn6bcMUI6soodj3WWKfgEO6JosKU1W5n3uky3WW9RlRjTg=="], + "@emnapi/core": ["@emnapi/core@1.8.1", "", { "dependencies": { "@emnapi/wasi-threads": "1.1.0", "tslib": "^2.4.0" } }, "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.8.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg=="], + + "@emnapi/wasi-threads": ["@emnapi/wasi-threads@1.1.0", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], @@ -135,6 +142,54 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], + "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.1", "", { "dependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1", "@tybys/wasm-util": "^0.10.1" } }, "sha512-p64ah1M1ld8xjWv3qbvFwHiFVWrq1yFvV4f7w+mzaqiR4IlSgkqhcRdHwsGgomwzBH51sRY4NEowLxnaBjcW/A=="], + + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], + + "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], + + "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + + "@oxc-resolver/binding-android-arm-eabi": ["@oxc-resolver/binding-android-arm-eabi@11.19.1", "", { "os": "android", "cpu": "arm" }, "sha512-aUs47y+xyXHUKlbhqHUjBABjvycq6YSD7bpxSW7vplUmdzAlJ93yXY6ZR0c1o1x5A/QKbENCvs3+NlY8IpIVzg=="], + + "@oxc-resolver/binding-android-arm64": ["@oxc-resolver/binding-android-arm64@11.19.1", "", { "os": "android", "cpu": "arm64" }, "sha512-oolbkRX+m7Pq2LNjr/kKgYeC7bRDMVTWPgxBGMjSpZi/+UskVo4jsMU3MLheZV55jL6c3rNelPl4oD60ggYmqA=="], + + "@oxc-resolver/binding-darwin-arm64": ["@oxc-resolver/binding-darwin-arm64@11.19.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-nUC6d2i3R5B12sUW4O646qD5cnMXf2oBGPLIIeaRfU9doJRORAbE2SGv4eW6rMqhD+G7nf2Y8TTJTLiiO3Q/dQ=="], + + "@oxc-resolver/binding-darwin-x64": ["@oxc-resolver/binding-darwin-x64@11.19.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-cV50vE5+uAgNcFa3QY1JOeKDSkM/9ReIcc/9wn4TavhW/itkDGrXhw9jaKnkQnGbjJ198Yh5nbX/Gr2mr4Z5jQ=="], + + "@oxc-resolver/binding-freebsd-x64": ["@oxc-resolver/binding-freebsd-x64@11.19.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-xZOQiYGFxtk48PBKff+Zwoym7ScPAIVp4c14lfLxizO2LTTTJe5sx9vQNGrBymrf/vatSPNMD4FgsaaRigPkqw=="], + + "@oxc-resolver/binding-linux-arm-gnueabihf": ["@oxc-resolver/binding-linux-arm-gnueabihf@11.19.1", "", { "os": "linux", "cpu": "arm" }, "sha512-lXZYWAC6kaGe/ky2su94e9jN9t6M0/6c+GrSlCqL//XO1cxi5lpAhnJYdyrKfm0ZEr/c7RNyAx3P7FSBcBd5+A=="], + + "@oxc-resolver/binding-linux-arm-musleabihf": ["@oxc-resolver/binding-linux-arm-musleabihf@11.19.1", "", { "os": "linux", "cpu": "arm" }, "sha512-veG1kKsuK5+t2IsO9q0DErYVSw2azvCVvWHnfTOS73WE0STdLLB7Q1bB9WR+yHPQM76ASkFyRbogWo1GR1+WbQ=="], + + "@oxc-resolver/binding-linux-arm64-gnu": ["@oxc-resolver/binding-linux-arm64-gnu@11.19.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-heV2+jmXyYnUrpUXSPugqWDRpnsQcDm2AX4wzTuvgdlZfoNYO0O3W2AVpJYaDn9AG4JdM6Kxom8+foE7/BcSig=="], + + "@oxc-resolver/binding-linux-arm64-musl": ["@oxc-resolver/binding-linux-arm64-musl@11.19.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-jvo2Pjs1c9KPxMuMPIeQsgu0mOJF9rEb3y3TdpsrqwxRM+AN6/nDDwv45n5ZrUnQMsdBy5gIabioMKnQfWo9ew=="], + + "@oxc-resolver/binding-linux-ppc64-gnu": ["@oxc-resolver/binding-linux-ppc64-gnu@11.19.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-vLmdNxWCdN7Uo5suays6A/+ywBby2PWBBPXctWPg5V0+eVuzsJxgAn6MMB4mPlshskYbppjpN2Zg83ArHze9gQ=="], + + "@oxc-resolver/binding-linux-riscv64-gnu": ["@oxc-resolver/binding-linux-riscv64-gnu@11.19.1", "", { "os": "linux", "cpu": "none" }, "sha512-/b+WgR+VTSBxzgOhDO7TlMXC1ufPIMR6Vj1zN+/x+MnyXGW7prTLzU9eW85Aj7Th7CCEG9ArCbTeqxCzFWdg2w=="], + + "@oxc-resolver/binding-linux-riscv64-musl": ["@oxc-resolver/binding-linux-riscv64-musl@11.19.1", "", { "os": "linux", "cpu": "none" }, "sha512-YlRdeWb9j42p29ROh+h4eg/OQ3dTJlpHSa+84pUM9+p6i3djtPz1q55yLJhgW9XfDch7FN1pQ/Vd6YP+xfRIuw=="], + + "@oxc-resolver/binding-linux-s390x-gnu": ["@oxc-resolver/binding-linux-s390x-gnu@11.19.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-EDpafVOQWF8/MJynsjOGFThcqhRHy417sRyLfQmeiamJ8qVhSKAn2Dn2VVKUGCjVB9C46VGjhNo7nOPUi1x6uA=="], + + "@oxc-resolver/binding-linux-x64-gnu": ["@oxc-resolver/binding-linux-x64-gnu@11.19.1", "", { "os": "linux", "cpu": "x64" }, "sha512-NxjZe+rqWhr+RT8/Ik+5ptA3oz7tUw361Wa5RWQXKnfqwSSHdHyrw6IdcTfYuml9dM856AlKWZIUXDmA9kkiBQ=="], + + "@oxc-resolver/binding-linux-x64-musl": ["@oxc-resolver/binding-linux-x64-musl@11.19.1", "", { "os": "linux", "cpu": "x64" }, "sha512-cM/hQwsO3ReJg5kR+SpI69DMfvNCp+A/eVR4b4YClE5bVZwz8rh2Nh05InhwI5HR/9cArbEkzMjcKgTHS6UaNw=="], + + "@oxc-resolver/binding-openharmony-arm64": ["@oxc-resolver/binding-openharmony-arm64@11.19.1", "", { "os": "none", "cpu": "arm64" }, "sha512-QF080IowFB0+9Rh6RcD19bdgh49BpQHUW5TajG1qvWHvmrQznTZZjYlgE2ltLXyKY+qs4F/v5xuX1XS7Is+3qA=="], + + "@oxc-resolver/binding-wasm32-wasi": ["@oxc-resolver/binding-wasm32-wasi@11.19.1", "", { "dependencies": { "@napi-rs/wasm-runtime": "^1.1.1" }, "cpu": "none" }, "sha512-w8UCKhX826cP/ZLokXDS6+milN8y4X7zidsAttEdWlVoamTNf6lhBJldaWr3ukTDiye7s4HRcuPEPOXNC432Vg=="], + + "@oxc-resolver/binding-win32-arm64-msvc": ["@oxc-resolver/binding-win32-arm64-msvc@11.19.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-nJ4AsUVZrVKwnU/QRdzPCCrO0TrabBqgJ8pJhXITdZGYOV28TIYystV1VFLbQ7DtAcaBHpocT5/ZJnF78YJPtQ=="], + + "@oxc-resolver/binding-win32-ia32-msvc": ["@oxc-resolver/binding-win32-ia32-msvc@11.19.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-EW+ND5q2Tl+a3pH81l1QbfgbF3HmqgwLfDfVithRFheac8OTcnbXt/JxqD2GbDkb7xYEqy1zNaVFRr3oeG8npA=="], + + "@oxc-resolver/binding-win32-x64-msvc": ["@oxc-resolver/binding-win32-x64-msvc@11.19.1", "", { "os": "win32", "cpu": "x64" }, "sha512-6hIU3RQu45B+VNTY4Ru8ppFwjVS/S5qwYyGhBotmjxfEKk41I2DlGtRfGJndZ5+6lneE2pwloqunlOyZuX/XAw=="], + "@preact/preset-vite": ["@preact/preset-vite@2.10.3", "", { "dependencies": { "@babel/plugin-transform-react-jsx": "^7.27.1", "@babel/plugin-transform-react-jsx-development": "^7.27.1", "@prefresh/vite": "^2.4.11", "@rollup/pluginutils": "^5.0.0", "babel-plugin-transform-hook-names": "^1.0.2", "debug": "^4.4.3", "picocolors": "^1.1.1", "vite-prerender-plugin": "^0.5.8" }, "peerDependencies": { "@babel/core": "7.x", "vite": "2.x || 3.x || 4.x || 5.x || 6.x || 7.x" } }, "sha512-1SiS+vFItpkNdBs7q585PSAIln0wBeBdcpJYbzPs1qipsb/FssnkUioNXuRsb8ZnU8YEQHr+3v8+/mzWSnTQmg=="], "@prefresh/babel-plugin": ["@prefresh/babel-plugin@0.5.3", "", {}, "sha512-57LX2SHs4BX2s1IwCjNzTE2OJeEepRCNf1VTEpbNcUyHfMO68eeOWGDIt4ob9aYlW6PEWZ1SuwNikuoIXANDtQ=="], @@ -199,8 +254,12 @@ "@tweenjs/tween.js": ["@tweenjs/tween.js@23.1.3", "", {}, "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA=="], + "@tybys/wasm-util": ["@tybys/wasm-util@0.10.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg=="], + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + "@types/node": ["@types/node@25.3.5", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA=="], + "@types/stats.js": ["@types/stats.js@0.17.4", "", {}, "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA=="], "@types/three": ["@types/three@0.165.0", "", { "dependencies": { "@tweenjs/tween.js": "~23.1.1", "@types/stats.js": "*", "@types/webxr": "*", "fflate": "~0.8.2", "meshoptimizer": "~0.18.1" } }, "sha512-AJK8JZAFNBF0kBXiAIl5pggYlzAGGA8geVYQXAcPCEDRbyA+oEjkpUBcJJrtNz6IiALwzGexFJGZG2yV3WsYBw=="], @@ -213,6 +272,8 @@ "boolbase": ["boolbase@1.0.0", "", {}, "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="], + "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], + "browserslist": ["browserslist@4.28.1", "", { "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", "electron-to-chromium": "^1.5.263", "node-releases": "^2.0.27", "update-browserslist-db": "^1.2.0" }, "bin": { "browserslist": "cli.js" } }, "sha512-ZC5Bd0LgJXgwGqUknZY/vkUQ04r8NXnJZ3yYi4vDmSiZmC/pdSN0NbNRPxZpbtO4uAfDUAFffO8IZoM3Gj8IkA=="], "caniuse-lite": ["caniuse-lite@1.0.30001777", "", {}, "sha512-tmN+fJxroPndC74efCdp12j+0rk0RHwV5Jwa1zWaFVyw2ZxAuPeG8ZgWC3Wz7uSjT3qMRQ5XHZ4COgQmsCMJAQ=="], @@ -243,20 +304,42 @@ "estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + "fast-glob": ["fast-glob@3.3.3", "", { "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", "glob-parent": "^5.1.2", "merge2": "^1.3.0", "micromatch": "^4.0.8" } }, "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg=="], + + "fastq": ["fastq@1.20.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw=="], + + "fd-package-json": ["fd-package-json@2.0.0", "", { "dependencies": { "walk-up-path": "^4.0.0" } }, "sha512-jKmm9YtsNXN789RS/0mSzOC1NUq9mkVd65vbSSVsKdjGvYXBuE4oWe2QOEoFeRmJg+lPuZxpmrfFclNhoRMneQ=="], + "fflate": ["fflate@0.8.2", "", {}, "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A=="], + "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + + "formatly": ["formatly@0.3.0", "", { "dependencies": { "fd-package-json": "^2.0.0" }, "bin": { "formatly": "bin/index.mjs" } }, "sha512-9XNj/o4wrRFyhSMJOvsuyMwy8aUfBaZ1VrqHVfohyXf0Sw0e+yfKG+xZaY3arGCOMdwFsqObtzVOc1gU9KiT9w=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], + "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], + "he": ["he@1.2.0", "", { "bin": { "he": "bin/he" } }, "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="], + "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], + + "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], + + "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + + "jiti": ["jiti@2.6.1", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ=="], + "js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], "jsesc": ["jsesc@3.1.0", "", { "bin": { "jsesc": "bin/jsesc" } }, "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA=="], "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "knip": ["knip@5.86.0", "", { "dependencies": { "@nodelib/fs.walk": "^1.2.3", "fast-glob": "^3.3.3", "formatly": "^0.3.0", "jiti": "^2.6.0", "minimist": "^1.2.8", "oxc-resolver": "^11.19.1", "picocolors": "^1.1.1", "picomatch": "^4.0.1", "smol-toml": "^1.5.2", "strip-json-comments": "5.0.3", "unbash": "^2.2.0", "yaml": "^2.8.2", "zod": "^4.1.11" }, "peerDependencies": { "@types/node": ">=18", "typescript": ">=5.0.4 <7" }, "bin": { "knip": "bin/knip.js", "knip-bun": "bin/knip-bun.js" } }, "sha512-tGpRCbP+L+VysXnAp1bHTLQ0k/SdC3M3oX18+Cpiqax1qdS25iuCPzpK8LVmAKARZv0Ijri81Wq09Rzk0JTl+Q=="], + "kolorist": ["kolorist@1.8.0", "", {}, "sha512-Y+60/zizpJ3HRH8DCss+q95yr6145JXZo46OTpFvDZWLfRCE4qChOyk1b26nMaNpfHHgxagk9dXT5OP0Tfe+dQ=="], "lru-cache": ["lru-cache@5.1.1", "", { "dependencies": { "yallist": "^3.0.2" } }, "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w=="], @@ -265,8 +348,14 @@ "marked": ["marked@12.0.2", "", { "bin": { "marked": "bin/marked.js" } }, "sha512-qXUm7e/YKFoqFPYPa3Ukg9xlI5cyAtGmyEIzMfW//m6kXwCy2Ps9DYf5ioijFKQ8qyuscrHoY04iJGctu2Kg0Q=="], + "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], + "meshoptimizer": ["meshoptimizer@0.18.1", "", {}, "sha512-ZhoIoL7TNV4s5B6+rx5mC//fw8/POGyNxS/DZyCJeiZ12ScLfVwRE/GfsxwiTkMYYD5DmK2/JXnEVXqL4rF+Sw=="], + "micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="], + + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], @@ -277,6 +366,8 @@ "nth-check": ["nth-check@2.1.1", "", { "dependencies": { "boolbase": "^1.0.0" } }, "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w=="], + "oxc-resolver": ["oxc-resolver@11.19.1", "", { "optionalDependencies": { "@oxc-resolver/binding-android-arm-eabi": "11.19.1", "@oxc-resolver/binding-android-arm64": "11.19.1", "@oxc-resolver/binding-darwin-arm64": "11.19.1", "@oxc-resolver/binding-darwin-x64": "11.19.1", "@oxc-resolver/binding-freebsd-x64": "11.19.1", "@oxc-resolver/binding-linux-arm-gnueabihf": "11.19.1", "@oxc-resolver/binding-linux-arm-musleabihf": "11.19.1", "@oxc-resolver/binding-linux-arm64-gnu": "11.19.1", "@oxc-resolver/binding-linux-arm64-musl": "11.19.1", "@oxc-resolver/binding-linux-ppc64-gnu": "11.19.1", "@oxc-resolver/binding-linux-riscv64-gnu": "11.19.1", "@oxc-resolver/binding-linux-riscv64-musl": "11.19.1", "@oxc-resolver/binding-linux-s390x-gnu": "11.19.1", "@oxc-resolver/binding-linux-x64-gnu": "11.19.1", "@oxc-resolver/binding-linux-x64-musl": "11.19.1", "@oxc-resolver/binding-openharmony-arm64": "11.19.1", "@oxc-resolver/binding-wasm32-wasi": "11.19.1", "@oxc-resolver/binding-win32-arm64-msvc": "11.19.1", "@oxc-resolver/binding-win32-ia32-msvc": "11.19.1", "@oxc-resolver/binding-win32-x64-msvc": "11.19.1" } }, "sha512-qE/CIg/spwrTBFt5aKmwe3ifeDdLfA2NESN30E42X/lII5ClF8V7Wt6WIJhcGZjp0/Q+nQ+9vgxGk//xZNX2hg=="], + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], @@ -285,32 +376,58 @@ "preact": ["preact@10.28.4", "", {}, "sha512-uKFfOHWuSNpRFVTnljsCluEFq57OKT+0QdOiQo8XWnQ/pSvg7OpX5eNOejELXJMWy+BwM2nobz0FkvzmnpCNsQ=="], + "queue-microtask": ["queue-microtask@1.2.3", "", {}, "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A=="], + + "reusify": ["reusify@1.1.0", "", {}, "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw=="], + "rollup": ["rollup@4.59.0", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.59.0", "@rollup/rollup-android-arm64": "4.59.0", "@rollup/rollup-darwin-arm64": "4.59.0", "@rollup/rollup-darwin-x64": "4.59.0", "@rollup/rollup-freebsd-arm64": "4.59.0", "@rollup/rollup-freebsd-x64": "4.59.0", "@rollup/rollup-linux-arm-gnueabihf": "4.59.0", "@rollup/rollup-linux-arm-musleabihf": "4.59.0", "@rollup/rollup-linux-arm64-gnu": "4.59.0", "@rollup/rollup-linux-arm64-musl": "4.59.0", "@rollup/rollup-linux-loong64-gnu": "4.59.0", "@rollup/rollup-linux-loong64-musl": "4.59.0", "@rollup/rollup-linux-ppc64-gnu": "4.59.0", "@rollup/rollup-linux-ppc64-musl": "4.59.0", "@rollup/rollup-linux-riscv64-gnu": "4.59.0", "@rollup/rollup-linux-riscv64-musl": "4.59.0", "@rollup/rollup-linux-s390x-gnu": "4.59.0", "@rollup/rollup-linux-x64-gnu": "4.59.0", "@rollup/rollup-linux-x64-musl": "4.59.0", "@rollup/rollup-openbsd-x64": "4.59.0", "@rollup/rollup-openharmony-arm64": "4.59.0", "@rollup/rollup-win32-arm64-msvc": "4.59.0", "@rollup/rollup-win32-ia32-msvc": "4.59.0", "@rollup/rollup-win32-x64-gnu": "4.59.0", "@rollup/rollup-win32-x64-msvc": "4.59.0", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg=="], + "run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="], + "semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="], "simple-code-frame": ["simple-code-frame@1.3.0", "", { "dependencies": { "kolorist": "^1.6.0" } }, "sha512-MB4pQmETUBlNs62BBeRjIFGeuy/x6gGKh7+eRUemn1rCFhqo7K+4slPqsyizCbcbYLnaYqaoZ2FWsZ/jN06D8w=="], + "smol-toml": ["smol-toml@1.6.0", "", {}, "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw=="], + "source-map": ["source-map@0.7.6", "", {}, "sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], "stack-trace": ["stack-trace@1.0.0-pre2", "", {}, "sha512-2ztBJRek8IVofG9DBJqdy2N5kulaacX30Nz7xmkYF6ale9WBVmIy6mFBchvGX7Vx/MyjBhx+Rcxqrj+dbOnQ6A=="], + "strip-json-comments": ["strip-json-comments@5.0.3", "", {}, "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw=="], + "three": ["three@0.165.0", "", {}, "sha512-cc96IlVYGydeceu0e5xq70H8/yoVT/tXBxV/W8A/U6uOq7DXc4/s1Mkmnu6SqoYGhSRWWYFOhVwvq6V0VtbplA=="], + "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + "unbash": ["unbash@2.2.0", "", {}, "sha512-X2wH19RAPZE3+ldGicOkoj/SIA83OIxcJ6Cuaw23hf8Xc6fQpvZXY0SftE2JgS0QhYLUG4uwodSI3R53keyh7w=="], + + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], + "update-browserslist-db": ["update-browserslist-db@1.2.3", "", { "dependencies": { "escalade": "^3.2.0", "picocolors": "^1.1.1" }, "peerDependencies": { "browserslist": ">= 4.21.0" }, "bin": { "update-browserslist-db": "cli.js" } }, "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w=="], "vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], "vite-prerender-plugin": ["vite-prerender-plugin@0.5.12", "", { "dependencies": { "kolorist": "^1.8.0", "magic-string": "0.x >= 0.26.0", "node-html-parser": "^6.1.12", "simple-code-frame": "^1.3.0", "source-map": "^0.7.4", "stack-trace": "^1.0.0-pre2" }, "peerDependencies": { "vite": "5.x || 6.x || 7.x" } }, "sha512-EiwhbMn+flg14EysbLTmZSzq8NGTxhytgK3bf4aGRF1evWLGwZiHiUJ1KZDvbxgKbMf2pG6fJWGEa3UZXOnR1g=="], + "walk-up-path": ["walk-up-path@4.0.0", "", {}, "sha512-3hu+tD8YzSLGuFYtPRb48vdhKMi0KQV5sn+uWr8+7dMEq/2G/dtLrdDinkLjqq5TIbIBjYJ4Ax/n3YiaW7QM8A=="], + "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], + "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], + + "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@prefresh/vite/@rollup/pluginutils": ["@rollup/pluginutils@4.2.1", "", { "dependencies": { "estree-walker": "^2.0.1", "picomatch": "^2.2.2" } }, "sha512-iKnFXr7NkdZAIHiIWE+BX5ULi/ucVFYWD6TbAV+rZctiRTY2PL6tsIKhoIOaoskiWAkgu+VsbXgUVDNLHf+InQ=="], + "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "@prefresh/vite/@rollup/pluginutils/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], } } diff --git a/frontend/package.json b/frontend/package.json index 503f6e0..1004a20 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -20,6 +20,7 @@ "@biomejs/biome": "^2.4.6", "@preact/preset-vite": "^2.8.1", "@types/three": "^0.165.0", + "knip": "^5.86.0", "typescript": "^5.4.5", "vite": "^5.2.11" } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 1aca081..9805446 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,61 +1,275 @@ -import { useCallback, useEffect } from "preact/hooks"; +import { useCallback, useEffect, useMemo, useRef, useState } from "preact/hooks"; import { AgentIndicator } from "./components/AgentIndicator"; +import { CardFeed } from "./components/CardFeed"; import { ControlBar, VoiceStatus } from "./components/Controls"; import { LogPanel } from "./components/LogPanel"; -import { ToastContainer } from "./components/Toast"; import { useAudioMeter } from "./hooks/useAudioMeter"; import { usePTT } from "./hooks/usePTT"; import { useWebRTC } from "./hooks/useWebRTC"; +import type { CardItem, CardMessageMetadata, JsonValue } from "./types"; -export function App() { - const rtc = useWebRTC(); - const audioLevel = useAudioMeter(rtc.remoteStream); +const SWIPE_THRESHOLD_PX = 64; +const SWIPE_DIRECTION_RATIO = 1.15; - const { agentStateOverride, handlePointerDown, handlePointerUp } = usePTT({ - connected: rtc.connected, - onSendPtt: (pressed) => rtc.sendJson({ type: "voice-ptt", pressed }), - onBootstrap: rtc.connect, - }); +interface AppRtcActions { + connect(): Promise; + sendJson( + msg: + | { type: "command"; command: string } + | { type: "card-response"; card_id: string; value: string } + | { type: "voice-ptt"; pressed: boolean; metadata?: CardMessageMetadata }, + ): void; + setTextOnly(enabled: boolean): void; + sendTextMessage(text: string, metadata?: CardMessageMetadata): Promise; + connected: boolean; + connecting: boolean; +} - const effectiveAgentState = agentStateOverride ?? rtc.agentState; +function buildCardMetadata(card: CardItem): CardMessageMetadata { + const metadata: CardMessageMetadata = { + card_id: card.serverId, + card_slot: card.slot, + card_title: card.title, + card_lane: card.lane, + card_template_key: card.templateKey, + card_context_summary: card.contextSummary, + card_response_value: card.responseValue, + }; + const liveContent = card.serverId + ? window.__nanobotGetCardLiveContent?.(card.serverId) + : undefined; + if (liveContent !== undefined) metadata.card_live_content = liveContent as JsonValue; + return metadata; +} - useEffect(() => { - document.addEventListener("pointerdown", handlePointerDown, { passive: false }); - document.addEventListener("pointerup", handlePointerUp, { passive: false }); - document.addEventListener("pointercancel", handlePointerUp, { passive: false }); - return () => { - document.removeEventListener("pointerdown", handlePointerDown); - document.removeEventListener("pointerup", handlePointerUp); - document.removeEventListener("pointercancel", handlePointerUp); - }; - }, [handlePointerDown, handlePointerUp]); +function AgentCardContext({ card, onClear }: { card: CardItem; onClear(): void }) { + return ( +
+
Using card
+
+
{card.title}
+ +
+
+ ); +} +function useSwipeHandlers( + composing: boolean, + view: "agent" | "feed", + setView: (view: "agent" | "feed") => void, + isInteractiveTarget: (target: EventTarget | null) => boolean, +) { + const swipeStartRef = useRef<{ x: number; y: number } | null>(null); + + const onSwipeStart = useCallback( + (e: Event) => { + const pe = e as PointerEvent; + if (composing) return; + if (pe.pointerType === "mouse" && pe.button !== 0) return; + if (isInteractiveTarget(pe.target)) return; + swipeStartRef.current = { x: pe.clientX, y: pe.clientY }; + }, + [composing, isInteractiveTarget], + ); + + const onSwipeEnd = useCallback( + (e: Event) => { + const pe = e as PointerEvent; + const start = swipeStartRef.current; + swipeStartRef.current = null; + if (!start || composing) return; + const dx = pe.clientX - start.x; + const dy = pe.clientY - start.y; + if (Math.abs(dx) < SWIPE_THRESHOLD_PX) return; + if (Math.abs(dx) < Math.abs(dy) * SWIPE_DIRECTION_RATIO) return; + if (view === "agent" && dx < 0) setView("feed"); + if (view === "feed" && dx > 0) setView("agent"); + }, + [composing, view, setView], + ); + + return { onSwipeStart, onSwipeEnd }; +} + +function useCardActions( + setView: (view: "agent" | "feed") => void, + setSelectedCardId: (cardId: string | null) => void, +) { + const handleAskCard = useCallback( + (card: CardItem) => { + if (!card.serverId) return; + setSelectedCardId(card.serverId); + setView("agent"); + }, + [setSelectedCardId, setView], + ); + + return { handleAskCard }; +} + +function useControlActions(rtc: AppRtcActions) { const handleReset = useCallback(async () => { + const confirmed = window.confirm("Clear the current conversation context and start fresh?"); + if (!confirmed) return; await rtc.connect(); rtc.sendJson({ type: "command", command: "reset" }); }, [rtc]); - const handleChoice = useCallback( - (requestId: string, value: string) => { - rtc.sendJson({ type: "ui-response", request_id: requestId, value }); + const handleToggleTextOnly = useCallback( + async (enabled: boolean) => { + rtc.setTextOnly(enabled); + if (enabled && !rtc.connected && !rtc.connecting) await rtc.connect(); }, [rtc], ); + return { handleReset, handleToggleTextOnly }; +} + +export function App() { + const rtc = useWebRTC(); + const remoteAudioLevel = useAudioMeter(rtc.remoteStream); + const audioLevel = rtc.textOnly ? 0 : remoteAudioLevel; + + const [view, setView] = useState<"agent" | "feed">("agent"); + const [composing, setComposing] = useState(false); + const [selectedCardId, setSelectedCardId] = useState(null); + const autoOpenedFeedRef = useRef(false); + + const selectedCard = useMemo( + () => + selectedCardId ? (rtc.cards.find((card) => card.serverId === selectedCardId) ?? null) : null, + [rtc.cards, selectedCardId], + ); + const selectedCardMetadata = useCallback( + () => (selectedCard ? buildCardMetadata(selectedCard) : undefined), + [selectedCard], + ); + + const { agentStateOverride, handlePointerDown, handlePointerMove, handlePointerUp } = usePTT({ + connected: rtc.connected && !rtc.textOnly, + onSendPtt: (pressed) => + rtc.sendJson({ type: "voice-ptt", pressed, metadata: selectedCardMetadata() }), + onBootstrap: rtc.connect, + }); + const effectiveAgentState = agentStateOverride ?? rtc.agentState; + + const isInteractiveTarget = useCallback((target: EventTarget | null): boolean => { + if (!(target instanceof Element)) return false; + return Boolean(target.closest("button,textarea,input,a,[data-no-swipe='1']")); + }, []); + const { onSwipeStart, onSwipeEnd } = useSwipeHandlers( + composing, + view, + setView, + isInteractiveTarget, + ); + + useEffect(() => { + document.addEventListener("pointerdown", handlePointerDown, { passive: false }); + document.addEventListener("pointermove", handlePointerMove, { passive: true }); + document.addEventListener("pointerup", handlePointerUp, { passive: false }); + document.addEventListener("pointercancel", handlePointerUp, { passive: false }); + return () => { + document.removeEventListener("pointerdown", handlePointerDown); + document.removeEventListener("pointermove", handlePointerMove); + document.removeEventListener("pointerup", handlePointerUp); + document.removeEventListener("pointercancel", handlePointerUp); + }; + }, [handlePointerDown, handlePointerMove, handlePointerUp]); + + useEffect(() => { + if (autoOpenedFeedRef.current || rtc.cards.length === 0) return; + autoOpenedFeedRef.current = true; + setView("feed"); + }, [rtc.cards.length]); + + useEffect(() => { + if (!selectedCardId) return; + if (rtc.cards.some((card) => card.serverId === selectedCardId)) return; + setSelectedCardId(null); + }, [rtc.cards, selectedCardId]); + + const { handleToggleTextOnly } = useControlActions(rtc); + const { handleAskCard } = useCardActions(setView, setSelectedCardId); + + const handleCardChoice = useCallback( + (cardId: string, value: string) => { + rtc.sendJson({ type: "card-response", card_id: cardId, value }); + }, + [rtc], + ); + const handleSendMessage = useCallback( + async (text: string) => { + await rtc.sendTextMessage(text, selectedCardMetadata()); + }, + [rtc, selectedCardMetadata], + ); + const handleResetWithSelection = useCallback(async () => { + const confirmed = window.confirm("Clear the current conversation context and start fresh?"); + if (!confirmed) return; + setSelectedCardId(null); + await rtc.connect(); + rtc.sendJson({ type: "command", command: "reset" }); + }, [rtc]); + return ( <> - - - {}} - onPointerUp={() => {}} - /> +
+
+
+ {view === "agent" && ( + + )} + {view === "agent" && selectedCard && ( + setSelectedCardId(null)} /> + )} + {view === "agent" && ( + + )} + {}} + onPointerUp={() => {}} + /> +
+
+ +
+
+
- ); } diff --git a/frontend/src/audioMeter.ts b/frontend/src/audioMeter.ts index 5c9a094..a61e7f7 100644 --- a/frontend/src/audioMeter.ts +++ b/frontend/src/audioMeter.ts @@ -4,7 +4,7 @@ const AudioContextCtor = (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext : undefined; -export interface AudioMeter { +interface AudioMeter { connect(stream: MediaStream): void; getLevel(): number; destroy(): void; diff --git a/frontend/src/components/AgentIndicator.tsx b/frontend/src/components/AgentIndicator.tsx index 56b5418..554c2f5 100644 --- a/frontend/src/components/AgentIndicator.tsx +++ b/frontend/src/components/AgentIndicator.tsx @@ -7,6 +7,7 @@ interface Props { connected: boolean; connecting: boolean; audioLevel: number; + viewActive: boolean; onPointerDown(): void; onPointerUp(): void; } @@ -16,6 +17,7 @@ export function AgentIndicator({ connected, connecting, audioLevel, + viewActive, onPointerDown, onPointerUp, }: Props) { @@ -49,7 +51,11 @@ export function AgentIndicator({ }, [audioLevel]); return ( -
+
(); + +function readCardState(script: HTMLScriptElement | null): Record { + const root = script?.closest("[data-nanobot-card-root]"); + if (!(root instanceof HTMLElement)) return {}; + + const stateEl = root.querySelector('script[data-card-state][type="application/json"]'); + if (!(stateEl instanceof HTMLScriptElement)) return {}; + + try { + const parsed = JSON.parse(stateEl.textContent || "{}"); + return typeof parsed === "object" && parsed !== null ? (parsed as Record) : {}; + } catch { + return {}; + } +} + +function resolveCardRoot(target: HTMLScriptElement | HTMLElement | null): HTMLElement | null { + if (!(target instanceof HTMLElement)) return null; + if (target.matches("[data-nanobot-card-root]")) return target; + return target.closest("[data-nanobot-card-root]"); +} + +function setCardLiveContent( + target: HTMLScriptElement | HTMLElement | null, + snapshot: JsonValue | null | undefined, +): void { + const root = resolveCardRoot(target); + const cardId = root?.dataset.cardId?.trim(); + if (!cardId) return; + if (snapshot === null || snapshot === undefined) { + cardLiveContentStore.delete(cardId); + return; + } + try { + cardLiveContentStore.set(cardId, JSON.parse(JSON.stringify(snapshot)) as JsonValue); + } catch { + cardLiveContentStore.delete(cardId); + } +} + +function getCardLiveContent(cardId: string | null | undefined): JsonValue | undefined { + const key = (cardId || "").trim(); + if (!key) return undefined; + const value = cardLiveContentStore.get(key); + if (value === undefined) return undefined; + try { + return JSON.parse(JSON.stringify(value)) as JsonValue; + } catch { + return undefined; + } +} + +function ensureCardStateHelper(): void { + if (!window.__nanobotGetCardState) { + window.__nanobotGetCardState = readCardState; + } + if (!window.__nanobotSetCardLiveContent) { + window.__nanobotSetCardLiveContent = setCardLiveContent; + } + if (!window.__nanobotGetCardLiveContent) { + window.__nanobotGetCardLiveContent = getCardLiveContent; + } +} + +declare global { + interface Window { + __nanobotGetCardState?: (script: HTMLScriptElement | null) => Record; + __nanobotSetCardLiveContent?: ( + target: HTMLScriptElement | HTMLElement | null, + snapshot: JsonValue | null | undefined, + ) => void; + __nanobotGetCardLiveContent?: (cardId: string | null | undefined) => JsonValue | undefined; + } +} + +const LANE_TITLES: Record = { + attention: "Attention", + work: "Work", + context: "Context", + history: "History", +}; +const LANE_ORDER: CardLane[] = ["attention", "work", "context", "history"]; + +interface CardProps { + card: CardItem; + onDismiss(id: number): void; + onChoice(cardId: string, value: string): void; + onAskCard(card: CardItem): void; +} + +function MoreIcon() { + return ( + + ); +} + +function CardTextBody({ card }: { card: CardItem }) { + const bodyRef = useRef(null); + + useEffect(() => { + ensureCardStateHelper(); + + const root = bodyRef.current; + if (!root) return; + const scripts = Array.from(root.querySelectorAll("script")); + for (const oldScript of scripts) { + const type = (oldScript.getAttribute("type") || "").trim().toLowerCase(); + if (!EXECUTABLE_SCRIPT_TYPES.has(type)) continue; + const runtimeScript = document.createElement("script"); + for (const attr of oldScript.attributes) runtimeScript.setAttribute(attr.name, attr.value); + runtimeScript.text = oldScript.textContent || ""; + oldScript.replaceWith(runtimeScript); + } + return () => { + window.__nanobotSetCardLiveContent?.(bodyRef.current, null); + }; + }, [card.id, card.content]); + + const looksLikeHtml = /^\s*<[a-zA-Z]/.test(card.content); + const html = looksLikeHtml ? card.content : (marked.parse(card.content) as string); + return
; +} + +function CardQuestionBody({ + card, + responding, + onChoice, +}: { + card: CardItem; + responding: boolean; + onChoice(cardId: string, value: string): void; +}) { + const canAnswer = card.state === "active" && !responding && !!card.serverId; + + return ( + <> + {card.content &&
{card.content}
} +
{card.question}
+
+ {(card.choices ?? []).map((label) => ( + + ))} +
+ {card.responseValue &&
Selected: {card.responseValue}
} + + ); +} + +function CardHeader({ + card, + onDismiss, + onAskCard, +}: { + card: CardItem; + onDismiss(): void; + onAskCard(card: CardItem): void; +}) { + const [menuOpen, setMenuOpen] = useState(false); + const menuRef = useRef(null); + + useEffect(() => { + if (!menuOpen) return; + + const handlePointerDown = (event: PointerEvent) => { + if (!(event.target instanceof Node)) return; + if (menuRef.current?.contains(event.target)) return; + setMenuOpen(false); + }; + + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") setMenuOpen(false); + }; + + document.addEventListener("pointerdown", handlePointerDown); + document.addEventListener("keydown", handleKeyDown); + return () => { + document.removeEventListener("pointerdown", handlePointerDown); + document.removeEventListener("keydown", handleKeyDown); + }; + }, [menuOpen]); + + return ( +
+ {card.kind !== "text" && ( +
+
+ {card.title && {card.title}} +
+
+ {card.state !== "active" && ( + {card.state} + )} +
+
+ )} +
+ + {menuOpen && ( + + )} +
+
+ ); +} + +function Card({ card, onDismiss, onChoice, onAskCard }: CardProps) { + const [dismissing, setDismissing] = useState(false); + const [responding, setResponding] = useState(false); + const timerRef = useRef | null>(null); + + useEffect(() => { + return () => { + if (timerRef.current) clearTimeout(timerRef.current); + }; + }, []); + + useEffect(() => { + if (card.state !== "active") setResponding(false); + }, [card.state]); + + const dismiss = () => { + if (dismissing) return; + setDismissing(true); + timerRef.current = setTimeout(() => onDismiss(card.id), 220); + }; + + return ( +
+ + {card.kind === "text" ? ( + + ) : ( + { + setResponding(true); + onChoice(cardId, value); + }} + /> + )} + {card.contextSummary && card.kind !== "text" && ( +
{card.contextSummary}
+ )} +
+ ); +} + +interface CardFeedProps { + cards: CardItem[]; + viewActive: boolean; + onDismiss(id: number): void; + onChoice(cardId: string, value: string): void; + onAskCard(card: CardItem): void; +} + +export function CardFeed({ cards, viewActive, onDismiss, onChoice, onAskCard }: CardFeedProps) { + const groups = useMemo( + () => + LANE_ORDER.map((lane) => ({ + lane, + title: LANE_TITLES[lane], + cards: cards.filter((card) => card.lane === lane), + })).filter((group) => group.cards.length > 0), + [cards], + ); + + return ( +
+ {groups.map((group) => ( +
+
{group.title}
+
+ {group.cards.map((card) => ( + + ))} +
+
+ ))} +
+ ); +} diff --git a/frontend/src/components/Controls.tsx b/frontend/src/components/Controls.tsx index 4242694..0d6d69c 100644 --- a/frontend/src/components/Controls.tsx +++ b/frontend/src/components/Controls.tsx @@ -3,6 +3,63 @@ interface VoiceStatusProps { visible: boolean; } +function SpeakerIcon() { + return ( + + ); +} + +function TextBubbleIcon() { + return ( + + ); +} + +function ResetIcon() { + return ( + + ); +} + export function VoiceStatus({ text, visible }: VoiceStatusProps) { return (
@@ -13,22 +70,52 @@ export function VoiceStatus({ text, visible }: VoiceStatusProps) { interface ControlBarProps { onReset(): void; + textOnly: boolean; + onToggleTextOnly(enabled: boolean): void; } -export function ControlBar({ onReset }: ControlBarProps) { +export function ControlBar({ onReset, textOnly, onToggleTextOnly }: ControlBarProps) { + const toggleLabel = textOnly ? "Text-only mode on" : "Voice mode on"; + return (
+
); diff --git a/frontend/src/components/FAB.tsx b/frontend/src/components/FAB.tsx new file mode 100644 index 0000000..72f74dd --- /dev/null +++ b/frontend/src/components/FAB.tsx @@ -0,0 +1,45 @@ +interface FABProps { + view: "agent" | "feed"; + unreadCount: number; + pttActive: boolean; +} + +function IconAgent() { + return ( + + ); +} + +function IconFeed() { + return ( + + ); +} + +export function FAB({ view, unreadCount, pttActive }: FABProps) { + const label = + view === "agent" ? "Switch to feed (hold to talk)" : "Switch to agent (hold to talk)"; + const badgeVisible = unreadCount > 0 && view === "agent"; + + return ( + + ); +} diff --git a/frontend/src/components/LogPanel.tsx b/frontend/src/components/LogPanel.tsx index 54a89fd..9392855 100644 --- a/frontend/src/components/LogPanel.tsx +++ b/frontend/src/components/LogPanel.tsx @@ -1,38 +1,199 @@ -import { useEffect, useRef } from "preact/hooks"; +import { useCallback, useEffect, useRef, useState } from "preact/hooks"; import type { LogLine } from "../types"; interface Props { lines: LogLine[]; + disabled: boolean; + onSendMessage(text: string): Promise; + onExpandChange?(expanded: boolean): void; } -export function LogPanel({ lines }: Props) { - const innerRef = useRef(null); +interface LogViewProps { + lines: LogLine[]; + scrollRef: { current: HTMLElement | null }; +} - // Scroll to top (newest line — column-reverse layout) after each update - useEffect(() => { - const el = innerRef.current?.parentElement; - if (el) el.scrollTop = 0; - }, [lines]); +function SendIcon() { + return ( + + ); +} + +function CloseIcon() { + return ( + + ); +} + +function formatLine(line: LogLine): string { + const time = line.timestamp ? new Date(line.timestamp).toLocaleTimeString() : ""; + const role = line.role.trim().toLowerCase(); + if (role === "nanobot") { + return `[${time}] ${line.text.replace(/^(?:nanobot|napbot)\b\s*[:>-]?\s*/i, "")}`; + } + if (role === "tool") { + return `[${time}] tool: ${line.text}`; + } + return `[${time}] ${line.role}: ${line.text}`; +} + +function LogCompose({ + disabled, + sending, + text, + setText, + onClose, + onSend, +}: { + disabled: boolean; + sending: boolean; + text: string; + setText(value: string): void; + onClose(): void; + onSend(): void; +}) { + const onKeyDown = useCallback( + (e: KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + onSend(); + } + if (e.key === "Escape") { + e.preventDefault(); + onClose(); + } + }, + [onClose, onSend], + ); return ( -
-
- {lines.map((line) => { - const time = line.timestamp ? new Date(line.timestamp).toLocaleTimeString() : ""; - const role = line.role.trim().toLowerCase(); - let text: string; - if (role === "nanobot") { - text = `[${time}] ${line.text.replace(/^(?:nanobot|napbot)\b\s*[:>-]?\s*/i, "")}`; - } else { - text = `[${time}] ${line.role}: ${line.text}`; - } - return ( -
- {text} -
- ); - })} +
+