Add frontend speech recognition

agent/languageDetect.ts

@@ -26,14 +26,6 @@ const USER_LANGUAGE_OUTPUT_SCHEMA = {
   },
 } as const;
 
-export function formatLanguagePrompt(language: UserLanguage | null) {
-  if (!language) {
-    return "Respond in the user's language.";
-  }
-
-  return `Respond in ${language.language} (${language.code}).`;
-}
-
 function parseUserLanguage(content: string | undefined): UserLanguage | null {
   if (!content) {
     return null;

agent/simpleAgent.ts

@@ -4,7 +4,6 @@ import {
   logger,
   type AdminUser,
   type CompletionAdapter,
-  type HttpExtra,
   type IAdminForth,
 } from "adminforth";
 import { BaseCallbackHandler } from "@langchain/core/callbacks/base";
@@ -26,8 +25,8 @@ export const contextSchema = z.object({
   userTimeZone: z.string(),
   sessionId: z.string(),
   turnId: z.string(),
+  abortSignal: z.custom<AbortSignal>().optional(),
   currentPage: z.custom<CurrentPageContext>().optional(),
-  httpExtra: z.custom<Partial<HttpExtra>>().optional(),
   emitToolCallEvent: z.custom<ToolCallEventSink>(),
 });
 
@@ -234,8 +233,8 @@ export async function callAgent(params: {
   sessionId: string;
   turnId: string;
   currentPage?: CurrentPageContext;
-  httpExtra?: Partial<HttpExtra>;
   userTimeZone: string;
+  abortSignal?: AbortSignal;
   emitToolCallEvent: ToolCallEventSink;
   sequenceDebugSink: SequenceDebugModelCallSink;
 }) {
@@ -253,8 +252,8 @@ export async function callAgent(params: {
     sessionId,
     turnId,
     currentPage,
-    httpExtra,
     userTimeZone,
+    abortSignal,
     emitToolCallEvent,
     sequenceDebugSink,
   } = params;
@@ -289,6 +288,7 @@ export async function callAgent(params: {
     streamMode: "messages",
     recursionLimit: 100,
     callbacks: [createAgentLlmMetricsLogger()],
+    signal: abortSignal,
     configurable: {
       thread_id: sessionId,
     },
@@ -297,8 +297,8 @@ export async function callAgent(params: {
       userTimeZone,
       sessionId,
       turnId,
+      abortSignal,
       currentPage,
-      httpExtra,
       emitToolCallEvent,
     },
   });

agent/systemPrompt.ts

@@ -1,10 +1,10 @@
-import type { AdminForthResource, IAdminForth } from "adminforth";
+import type { AdminForthResource, AdminUser, IAdminForth } from "adminforth";
+import type { UserLanguage } from "./languageDetect.js";
 import {
   listBundledSkillManifests,
   listProjectSkillManifests,
   type AgentSkillManifest,
 } from "./skills/registry.js";
-import { ALWAYS_AVAILABLE_API_TOOL_NAMES } from "./tools/index.js";
 
 export const DEFAULT_AGENT_SYSTEM_PROMPT = [
   "You are helpful AI Assistant for Admin Panel.",
@@ -46,6 +46,39 @@ export function appendCustomSystemPrompt(
   return `${systemPrompt}\n\n${normalizedCustomSystemPrompt}`;
 }
 
+function formatLanguagePrompt(language: UserLanguage | null) {
+  if (!language) {
+    return "Respond in the user's language.";
+  }
+
+  return `Respond in ${language.language} (${language.code}).`;
+}
+
+function formatAdminUserPrompt(adminUser: AdminUser, usernameField: string) {
+  const adminUserContext = {
+    id: adminUser.pk,
+    email: adminUser.dbUser[usernameField],
+  };
+  return [
+    "Current admin user context:",
+    JSON.stringify(adminUserContext, null, 2),
+    "Use this admin user email when the user asks to send information to themselves, the current admin, or the logged-in user.",
+  ].join("\n");
+}
+
+export function buildAgentTurnSystemPrompt(input: {
+  agentSystemPrompt: string;
+  adminUser: AdminUser;
+  usernameField: string;
+  userLanguage: UserLanguage | null;
+}) {
+  return [
+    input.agentSystemPrompt,
+    formatAdminUserPrompt(input.adminUser, input.usernameField),
+    formatLanguagePrompt(input.userLanguage),
+  ].join("\n\n");
+}
+
 function formatResources(resources: AdminForthResource[]) {
   return resources
     .map((resource) => `- resourceId: ${resource.resourceId}\n  label: ${resource.label}`)
@@ -66,7 +99,6 @@ export async function buildAgentSystemPrompt(
     listProjectSkillManifests(adminforth.config.customization.customComponentsDir),
     listBundledSkillManifests(),
   ]);
-  const alwaysAvailableTools = ALWAYS_AVAILABLE_API_TOOL_NAMES.join(", ");
   const adminBasePath = adminforth.config.baseUrlSlashed;
   const hiddenResourceIdSet = new Set(hiddenResourceIds);
   const visibleResources = adminforth.config.resources.filter(
@@ -76,7 +108,6 @@ export async function buildAgentSystemPrompt(
     DEFAULT_AGENT_SYSTEM_PROMPT,
     `ADMIN_BASE_PATH: ${adminBasePath}`,
     `List of resources:\n${formatResources(visibleResources)}`,
-    `You have always-available base tools: ${alwaysAvailableTools}.`,
     primarySkills.length > 0
       ? `You have primary skills set:\n${formatSkills(primarySkills, "skill_name")}`
       : "",

agent/toolCallEvents.ts

@@ -1,6 +1,5 @@
 import { randomUUID } from "crypto";
 import YAML from "yaml";
-import { serializeUnknownError } from "../apiBasedTools.js";
 
 export type ToolCallEvent =
   | {
@@ -61,6 +60,36 @@ function sanitizeToolCallOutputForDebug(output: unknown) {
   );
 }
 
+function serializeErrorForDebug(error: unknown): unknown {
+  if (!(error instanceof Error)) {
+    return error;
+  }
+
+  const errorWithCause = error as Error & { cause?: unknown };
+  const serialized: Record<string, unknown> = {
+    name: error.name,
+    message: error.message,
+  };
+
+  if (error.stack) {
+    serialized.stack = error.stack;
+  }
+
+  if (errorWithCause.cause !== undefined) {
+    serialized.cause = serializeErrorForDebug(errorWithCause.cause);
+  }
+
+  for (const key of Object.getOwnPropertyNames(error)) {
+    if (key in serialized) {
+      continue;
+    }
+
+    serialized[key] = (error as unknown as Record<string, unknown>)[key];
+  }
+
+  return serialized;
+}
+
 export function createToolCallTracker(params: {
   emit: ToolCallEventSink;
   toolCallId?: string;
@@ -99,7 +128,7 @@ export function createToolCallTracker(params: {
         phase: "end",
         durationMs: Date.now() - startedAt,
         output: null,
-        error: YAML.stringify(serializeUnknownError(error)).trimEnd(),
+        error: YAML.stringify(serializeErrorForDebug(error)).trimEnd(),
       });
     },
   };

agent/tools/apiTool.ts

@@ -51,7 +51,7 @@ export function createApiTool(toolName: string, apiBasedTool: ApiBasedTool) {
       const normalizedInput = (input ?? {}) as Record<string, unknown>;
       return apiBasedTool.call({
         adminUser: runtime.context.adminUser,
-        httpExtra: runtime.context.httpExtra,
+        abortSignal: runtime.context.abortSignal,
         inputs: normalizedInput,
         userTimeZone: runtime.context.userTimeZone,
       });

agentResponseEvents.ts

@@ -0,0 +1,197 @@
+import { randomUUID } from "crypto";
+
+import type { ToolCallEvent } from "./agent/toolCallEvents.js";
+
+type AgentEventStreamResponse = {
+  writeHead: (statusCode: number, headers: Record<string, string>) => void;
+  write: (chunk: string) => unknown;
+  end: () => unknown;
+  writableEnded: boolean;
+  destroyed: boolean;
+};
+
+type AgentEventStreamOptions = {
+  vercelAiUiMessageStream?: boolean;
+  closeActiveBlockOnToolStart?: boolean;
+};
+
+export function createAgentEventStream(
+  res: AgentEventStreamResponse,
+  options: AgentEventStreamOptions = {},
+) {
+  let isStreamClosed = false;
+  let activeBlock: { type: "text" | "reasoning"; id: string } | null = null;
+
+  res.writeHead(200, {
+    "Content-Type": "text/event-stream",
+    "Cache-Control": "no-cache",
+    "Connection": "keep-alive",
+    ...(options.vercelAiUiMessageStream
+      ? { "x-vercel-ai-ui-message-stream": "v1" }
+      : {}),
+  });
+
+  const stream = {
+    send(obj: unknown) {
+      if (isStreamClosed || res.writableEnded || res.destroyed) {
+        return;
+      }
+
+      res.write(`data: ${JSON.stringify(obj)}\n\n`);
+    },
+
+    endActiveBlock() {
+      if (!activeBlock) {
+        return;
+      }
+
+      stream.send({
+        type: `${activeBlock.type}-end`,
+        id: activeBlock.id,
+      });
+
+      activeBlock = null;
+    },
+
+    startBlock(type: "text" | "reasoning") {
+      if (activeBlock?.type === type) {
+        return activeBlock.id;
+      }
+
+      stream.endActiveBlock();
+
+      const id = randomUUID();
+      activeBlock = { type, id };
+
+      stream.send({
+        type: `${type}-start`,
+        id,
+      });
+
+      return id;
+    },
+
+    start(messageId: string) {
+      stream.send({
+        type: "start",
+        messageId,
+      });
+    },
+
+    textDelta(delta: string) {
+      const textId = stream.startBlock("text");
+      stream.send({
+        type: "text-delta",
+        id: textId,
+        delta,
+      });
+    },
+
+    reasoningDelta(delta: string) {
+      const reasoningId = stream.startBlock("reasoning");
+      stream.send({
+        type: "reasoning-delta",
+        id: reasoningId,
+        delta,
+      });
+    },
+
+    toolCall(event: ToolCallEvent) {
+      if (options.closeActiveBlockOnToolStart && event.phase === "start") {
+        stream.endActiveBlock();
+      }
+
+      stream.send({
+        type: "data-tool-call",
+        data: event,
+      });
+    },
+
+    transcript(text: string, language?: string) {
+      stream.send({
+        type: "transcript",
+        data: {
+          text,
+          language,
+        },
+      });
+    },
+
+    response(text: string, sessionId: string, turnId: string) {
+      stream.send({
+        type: "response",
+        data: {
+          text,
+          sessionId,
+          turnId,
+        },
+      });
+    },
+
+    speechResponse(
+      transcript: { text: string; language?: string },
+      response: { text: string },
+      sessionId: string,
+      turnId: string,
+    ) {
+      stream.send({
+        type: "speech-response",
+        data: {
+          transcript,
+          response,
+          sessionId,
+          turnId,
+        },
+      });
+    },
+
+    audioStart(mimeType: string, format: string) {
+      stream.send({
+        type: "audio-start",
+        data: {
+          mimeType,
+          format,
+        },
+      });
+    },
+
+    audioDelta(value: Uint8Array) {
+      stream.send({
+        type: "audio-delta",
+        data: {
+          base64: Buffer.from(value).toString("base64"),
+        },
+      });
+    },
+
+    audioDone() {
+      stream.send({
+        type: "audio-done",
+      });
+    },
+
+    error(error: string) {
+      stream.send({
+        type: "error",
+        error,
+      });
+    },
+
+    end() {
+      if (isStreamClosed || res.writableEnded || res.destroyed) {
+        return;
+      }
+
+      stream.endActiveBlock();
+      stream.send({
+        type: "finish",
+      });
+
+      res.write("data: [DONE]\n\n");
+      isStreamClosed = true;
+      res.end();
+    },
+  };
+
+  return stream;
+}