diff --git a/command-snapshot.json b/command-snapshot.json index cf97ec3f..94f0d57c 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -254,6 +254,14 @@ ], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:trace:delete", + "flagAliases": [], + "flagChars": ["a"], + "flags": ["agent", "flags-dir", "json", "no-prompt", "older-than", "session-id"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:trace:list", @@ -262,6 +270,14 @@ "flags": ["agent", "flags-dir", "json", "session-id", "since"], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:trace:read", + "flagAliases": [], + "flagChars": ["d", "f", "s", "t"], + "flags": ["dimension", "flags-dir", "format", "json", "session-id", "turn"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:validate:authoring-bundle", diff --git a/messages/agent.trace.delete.md b/messages/agent.trace.delete.md new file mode 100644 index 00000000..32f23c6d --- /dev/null +++ b/messages/agent.trace.delete.md @@ -0,0 +1,87 @@ +# summary + +Delete agent preview trace files. + +# description + +Deletes trace files recorded during agent preview sessions. By default, shows a preview of what will be deleted and prompts for confirmation. Use --no-prompt to skip confirmation. + +Without filters, deletes all traces for all agents and sessions. Use flags to narrow the scope: filter by agent name (--agent), by session (--session-id), or by age (--older-than). + +# flags.agent.summary + +Only delete traces for this agent name (substring match). Matches against the name used when starting the session, whether that's an authoring bundle or a published agent API name. + +# flags.session-id.summary + +Only delete traces from this session ID. + +# flags.older-than.summary + +Only delete traces older than this duration. Accepts a number followed by a unit: m/minutes, h/hours, d/days, w/weeks (e.g. 7d, 24h, 2w). + +# flags.no-prompt.summary + +Skip the confirmation prompt and delete immediately. + +# error.invalidOlderThan + +Invalid --older-than value: '%s'. Use a number followed by a unit: m/minutes, h/hours, d/days, w/weeks (e.g. 7d, 24h, 30m, 2w). + +# prompt.confirm + +Delete %s trace file(s)? This cannot be undone. + +# output.noneFound + +No trace files matched the specified filters. + +# output.preview + +Found %s trace file(s) to delete: + +# output.cancelled + +Deletion cancelled. + +# output.deleted + +Deleted %s trace file(s). + +# output.tableHeader.agent + +Agent + +# output.tableHeader.sessionId + +Session ID + +# output.tableHeader.planId + +Plan ID + +# examples + +- Delete all traces for all agents and sessions (with confirmation prompt): + + <%= config.bin %> <%= command.id %> + +- Delete all traces for a specific agent: + + <%= config.bin %> <%= command.id %> --agent My_Agent + +- Delete traces from a specific session: + + <%= config.bin %> <%= command.id %> --session-id + +- Delete traces older than 7 days: + + <%= config.bin %> <%= command.id %> --older-than 7d + +- Delete traces older than 24 hours for a specific agent, no prompt: + + <%= config.bin %> <%= command.id %> --agent My_Agent --older-than 24h --no-prompt + +- Delete all traces without confirmation: + + <%= config.bin %> <%= command.id %> --no-prompt diff --git a/messages/agent.trace.read.md b/messages/agent.trace.read.md new file mode 100644 index 00000000..f6fb1de3 --- /dev/null +++ b/messages/agent.trace.read.md @@ -0,0 +1,171 @@ +# summary + +Read and analyze trace files from an agent preview session. + +# description + +Reads trace files recorded during an agent preview session and outputs them in one of three formats. + +**--format summary** (default): A per-turn narrative showing topic routing, actions executed, and the agent's response. Use this to quickly understand what happened in a session. + +**--format detail**: Diagnostic drill-down into a specific dimension (--dimension required). Filters output to only the trace steps relevant to that dimension, minimizing noise. + +**--format raw**: Unprocessed trace JSON. Use this as a fallback when the trace schema has changed or you need to perform custom analysis. + +Available dimensions for --format detail: actions, grounding, routing, errors. + +Use --turn N to scope output to a single conversation turn. + +# flags.session-id.summary + +Session ID to read traces for. + +# flags.format.summary + +Output format: summary (default), detail, or raw. Use detail with --dimension to drill into a specific aspect of the trace. + +# flags.dimension.summary + +Dimension to drill into when using --format detail. One of: actions, grounding, routing, errors. Required when --format is detail. + +# flags.turn.summary + +Scope output to this conversation turn number. + +# error.detailRequiresDimension + +--format detail requires --dimension. Specify one of: actions, grounding, routing, errors. + +# error.sessionNotFound + +Session '%s' was not found in the local session cache. Run "sf agent trace list" to see available sessions. + +# error.turnIndexNotFound + +No turn index found for session '%s'. Cannot filter by --turn without a turn index. + +# error.turnNotFound + +Turn %s was not found in session '%s'. + +# error.parseFailedAll + +Trace parsing failed for all files: %s. The trace schema may have changed. Try --format raw to access unprocessed trace data. + +# warn.dimensionIgnored + +--dimension is ignored when --format is '%s'. Use --format detail to drill into a dimension. + +# warn.parseFailed + +Trace parsing failed for some files (skipped): %s. Try --format raw to access unprocessed trace data. + +# output.empty + +No traces found for this session. + +# output.emptyDimension + +No '%s' data found in the traces for this session. + +# output.tableHeader.turn + +Turn + +# output.tableHeader.topic + +Topic + +# output.tableHeader.userInput + +User Input + +# output.tableHeader.agentResponse + +Agent Response + +# output.tableHeader.actionsExecuted + +Actions Executed + +# output.tableHeader.latencyMs + +Latency + +# output.tableHeader.error + +Error + +# output.tableHeader.action + +Action + +# output.tableHeader.input + +Input + +# output.tableHeader.output + +Output + +# output.tableHeader.prompt + +Prompt + +# output.tableHeader.response + +Response + +# output.tableHeader.intent + +Intent + +# output.tableHeader.fromTopic + +From Topic + +# output.tableHeader.toTopic + +To Topic + +# output.tableHeader.source + +Source + +# output.tableHeader.errorCode + +Error Code + +# output.tableHeader.message + +Message + +# examples + +- Show a session summary (all turns): + + <%= config.bin %> <%= command.id %> --session-id + +- Show summary for a single turn: + + <%= config.bin %> <%= command.id %> --session-id --turn 2 + +- Drill into action execution across all turns: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension actions + +- Drill into routing decisions for a specific turn: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension routing --turn 1 + +- Show all errors across the session: + + <%= config.bin %> <%= command.id %> --session-id --format detail --dimension errors + +- Output raw trace JSON for custom parsing: + + <%= config.bin %> <%= command.id %> --session-id --format raw + +- Return results as JSON: + + <%= config.bin %> <%= command.id %> --session-id --json diff --git a/schemas/agent-trace-delete.json b/schemas/agent-trace-delete.json new file mode 100644 index 00000000..716fea24 --- /dev/null +++ b/schemas/agent-trace-delete.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTraceDeleteResult", + "definitions": { + "AgentTraceDeleteResult": { + "type": "array", + "items": { + "type": "object", + "properties": { + "agent": { + "type": "string" + }, + "sessionId": { + "type": "string" + }, + "planId": { + "type": "string" + }, + "path": { + "type": "string" + } + }, + "required": ["agent", "sessionId", "planId", "path"], + "additionalProperties": false + } + } + } +} diff --git a/schemas/agent-trace-read.json b/schemas/agent-trace-read.json new file mode 100644 index 00000000..e8162963 --- /dev/null +++ b/schemas/agent-trace-read.json @@ -0,0 +1,466 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentTraceReadResult", + "definitions": { + "AgentTraceReadResult": { + "type": "object", + "properties": { + "sessionId": { + "type": "string" + }, + "format": { + "type": "string", + "enum": ["summary", "detail", "raw"] + }, + "dimension": { + "$ref": "#/definitions/Dimension" + }, + "turns": { + "type": "array", + "items": { + "$ref": "#/definitions/TurnSummary" + } + }, + "detail": { + "type": "array", + "items": { + "$ref": "#/definitions/DimensionRow" + } + }, + "raw": { + "type": "array", + "items": { + "$ref": "#/definitions/PlannerResponse" + } + } + }, + "required": ["sessionId", "format"], + "additionalProperties": false + }, + "Dimension": { + "type": "string", + "enum": ["actions", "grounding", "routing", "errors"] + }, + "TurnSummary": { + "type": "object", + "properties": { + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "userInput": { + "type": "string" + }, + "agentResponse": { + "type": "string" + }, + "actionsExecuted": { + "type": "array", + "items": { + "type": "string" + } + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": ["string", "null"] + } + }, + "required": ["turn", "planId", "topic", "userInput", "agentResponse", "actionsExecuted", "latencyMs", "error"], + "additionalProperties": false + }, + "DimensionRow": { + "anyOf": [ + { + "$ref": "#/definitions/ActionsRow" + }, + { + "$ref": "#/definitions/GroundingRow" + }, + { + "$ref": "#/definitions/RoutingRow" + }, + { + "$ref": "#/definitions/ErrorsRow" + } + ] + }, + "ActionsRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "actions" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "action": { + "type": "string" + }, + "input": { + "type": "string" + }, + "output": { + "type": "string" + }, + "latencyMs": { + "type": "number" + }, + "error": { + "type": ["string", "null"] + } + }, + "required": ["dimension", "turn", "planId", "action", "input", "output", "latencyMs", "error"], + "additionalProperties": false + }, + "GroundingRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "grounding" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "response": { + "type": "string" + }, + "latencyMs": { + "type": "number" + } + }, + "required": ["dimension", "turn", "planId", "prompt", "response", "latencyMs"], + "additionalProperties": false + }, + "RoutingRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "routing" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "fromTopic": { + "type": "string" + }, + "toTopic": { + "type": "string" + }, + "intent": { + "type": "string" + } + }, + "required": ["dimension", "turn", "planId", "fromTopic", "toTopic", "intent"], + "additionalProperties": false + }, + "ErrorsRow": { + "type": "object", + "properties": { + "dimension": { + "type": "string", + "const": "errors" + }, + "turn": { + "type": "number" + }, + "planId": { + "type": "string" + }, + "source": { + "type": "string" + }, + "errorCode": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": ["dimension", "turn", "planId", "source", "errorCode", "message"], + "additionalProperties": false + }, + "PlannerResponse": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "PlanSuccessResponse" + }, + "planId": { + "type": "string" + }, + "sessionId": { + "type": "string" + }, + "intent": { + "type": "string" + }, + "topic": { + "type": "string" + }, + "plan": { + "type": "array", + "items": { + "$ref": "#/definitions/PlanStep" + } + } + }, + "required": ["type", "planId", "sessionId", "intent", "topic", "plan"], + "additionalProperties": false + }, + "PlanStep": { + "anyOf": [ + { + "$ref": "#/definitions/UserInputStep" + }, + { + "$ref": "#/definitions/LLMExecutionStep" + }, + { + "$ref": "#/definitions/UpdateTopicStep" + }, + { + "$ref": "#/definitions/EventStep" + }, + { + "$ref": "#/definitions/FunctionStep" + }, + { + "$ref": "#/definitions/PlannerResponseStep" + } + ] + }, + "UserInputStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "UserInputStep" + }, + "message": { + "type": "string" + } + }, + "required": ["type", "message"], + "additionalProperties": false + }, + "LLMExecutionStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "LLMExecutionStep" + }, + "promptName": { + "type": "string" + }, + "promptContent": { + "type": "string" + }, + "promptResponse": { + "type": "string" + }, + "executionLatency": { + "type": "number" + }, + "startExecutionTime": { + "type": "number" + }, + "endExecutionTime": { + "type": "number" + } + }, + "required": [ + "type", + "promptName", + "promptContent", + "promptResponse", + "executionLatency", + "startExecutionTime", + "endExecutionTime" + ], + "additionalProperties": false + }, + "UpdateTopicStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "UpdateTopicStep" + }, + "topic": { + "type": "string" + }, + "description": { + "type": "string" + }, + "job": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": { + "type": "string" + } + }, + "availableFunctions": { + "type": "array", + "items": {} + } + }, + "required": ["type", "topic", "description", "job", "instructions", "availableFunctions"], + "additionalProperties": false + }, + "EventStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "EventStep" + }, + "eventName": { + "type": "string" + }, + "isError": { + "type": "boolean" + }, + "payload": { + "type": "object", + "properties": { + "oldTopic": { + "type": "string" + }, + "newTopic": { + "type": "string" + } + }, + "required": ["oldTopic", "newTopic"], + "additionalProperties": false + } + }, + "required": ["type", "eventName", "isError", "payload"], + "additionalProperties": false + }, + "FunctionStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "FunctionStep" + }, + "function": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "input": { + "type": "object", + "additionalProperties": {} + }, + "output": { + "type": "object", + "additionalProperties": {} + } + }, + "required": ["name", "input", "output"], + "additionalProperties": false + }, + "executionLatency": { + "type": "number" + }, + "startExecutionTime": { + "type": "number" + }, + "endExecutionTime": { + "type": "number" + } + }, + "required": ["type", "function", "executionLatency", "startExecutionTime", "endExecutionTime"], + "additionalProperties": false + }, + "PlannerResponseStep": { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "PlannerResponseStep" + }, + "message": { + "type": "string" + }, + "responseType": { + "type": "string" + }, + "isContentSafe": { + "type": "boolean" + }, + "safetyScore": { + "type": "object", + "properties": { + "safety_score": { + "type": "number" + }, + "category_scores": { + "type": "object", + "properties": { + "toxicity": { + "type": "number" + }, + "hate": { + "type": "number" + }, + "identity": { + "type": "number" + }, + "violence": { + "type": "number" + }, + "physical": { + "type": "number" + }, + "sexual": { + "type": "number" + }, + "profanity": { + "type": "number" + }, + "biased": { + "type": "number" + } + }, + "required": ["toxicity", "hate", "identity", "violence", "physical", "sexual", "profanity", "biased"], + "additionalProperties": false + } + }, + "required": ["safety_score", "category_scores"], + "additionalProperties": false + } + }, + "required": ["type", "message", "responseType", "isContentSafe", "safetyScore"], + "additionalProperties": false + } + } +} diff --git a/src/agentSessionScanner.ts b/src/agentSessionScanner.ts new file mode 100644 index 00000000..1937bc75 --- /dev/null +++ b/src/agentSessionScanner.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { type SfProject } from '@salesforce/core'; +import { listCachedPreviewSessions } from '@salesforce/agents'; + +export type AgentSession = { agentId: string; displayName: string; sessionId: string }; + +/** + * Returns all sessions that have trace data on disk — both active sessions (still in the + * cache index) and ended sessions (removed from the index by `agent preview end` but whose + * trace directories remain under .sfdx/agents//sessions//). + */ +export async function listAllAgentSessions(project: SfProject): Promise { + const cached = await listCachedPreviewSessions(project); + const active: AgentSession[] = cached.flatMap(({ agentId, displayName, sessions }) => + sessions.map(({ sessionId }) => ({ agentId, displayName: displayName ?? agentId, sessionId })) + ); + + const seen = new Set(active.map((s) => s.sessionId)); + + const agentsDir = join(project.getPath(), '.sfdx', 'agents'); + const ended: AgentSession[] = []; + try { + const agentDirs = await readdir(agentsDir, { withFileTypes: true }); + for (const agentEnt of agentDirs) { + if (!agentEnt.isDirectory()) continue; + const sessionsDir = join(agentsDir, agentEnt.name, 'sessions'); + try { + // eslint-disable-next-line no-await-in-loop + const sessionDirs = await readdir(sessionsDir, { withFileTypes: true }); + for (const sessEnt of sessionDirs) { + if (!sessEnt.isDirectory() || seen.has(sessEnt.name)) continue; + ended.push({ agentId: agentEnt.name, displayName: agentEnt.name, sessionId: sessEnt.name }); + seen.add(sessEnt.name); + } + } catch { + // no sessions dir for this agent + } + } + } catch { + // no .sfdx/agents dir yet + } + + return [...active, ...ended]; +} diff --git a/src/commands/agent/trace/delete.ts b/src/commands/agent/trace/delete.ts new file mode 100644 index 00000000..9e51f843 --- /dev/null +++ b/src/commands/agent/trace/delete.ts @@ -0,0 +1,139 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { unlink } from 'node:fs/promises'; +import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, SfError } from '@salesforce/core'; +import { listSessionTraces, type TraceFileInfo } from '@salesforce/agents'; +import yesNoOrCancel from '../../../yes-no-cancel.js'; +import { listAllAgentSessions } from '../../../agentSessionScanner.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.trace.delete'); + +const DURATION_RE = /^(\d+)(d|h|m|w|days?|hours?|minutes?|weeks?)$/i; +const UNIT_MS: Record = { + m: 60_000, + minute: 60_000, + minutes: 60_000, + h: 3_600_000, + hour: 3_600_000, + hours: 3_600_000, + d: 86_400_000, + day: 86_400_000, + days: 86_400_000, + w: 604_800_000, + week: 604_800_000, + weeks: 604_800_000, +}; + +export type AgentTraceDeleteResult = Array<{ + agent: string; + sessionId: string; + planId: string; + path: string; +}>; + +export default class AgentTraceDelete extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly errorCodes = toHelpSection('ERROR CODES', { + 'Succeeded (0)': 'Traces deleted successfully (or no traces matched).', + }); + + public static readonly flags = { + agent: Flags.string({ + summary: messages.getMessage('flags.agent.summary'), + char: 'a', + }), + 'session-id': Flags.string({ + summary: messages.getMessage('flags.session-id.summary'), + }), + 'older-than': Flags.custom({ + summary: messages.getMessage('flags.older-than.summary'), + // eslint-disable-next-line @typescript-eslint/require-await + parse: async (raw): Promise => { + const match = DURATION_RE.exec(raw); + if (!match) { + throw new SfError(messages.getMessage('error.invalidOlderThan', [raw]), 'InvalidDuration'); + } + const ms = parseInt(match[1], 10) * UNIT_MS[match[2].toLowerCase()]; + return new Date(Date.now() - ms); + }, + })(), + 'no-prompt': Flags.boolean({ + summary: messages.getMessage('flags.no-prompt.summary'), + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTraceDelete); + + const agentFilter = flags.agent?.toLowerCase(); + const allSessions = await listAllAgentSessions(this.project!); + + const candidates: AgentTraceDeleteResult = []; + for (const { agentId, displayName, sessionId } of allSessions) { + if (agentFilter && !displayName.toLowerCase().includes(agentFilter)) continue; + if (flags['session-id'] && sessionId !== flags['session-id']) continue; + + // eslint-disable-next-line no-await-in-loop + let traces: TraceFileInfo[] = await listSessionTraces(agentId, sessionId); + + if (flags['older-than']) { + traces = traces.filter((t) => t.mtime < flags['older-than']!); + } + + for (const t of traces) { + candidates.push({ agent: displayName, sessionId, planId: t.planId, path: t.path }); + } + } + + if (candidates.length === 0) { + this.log(messages.getMessage('output.noneFound')); + return []; + } + + if (!flags['no-prompt']) { + this.log(messages.getMessage('output.preview', [candidates.length])); + this.table({ + data: candidates.map((c) => ({ agent: c.agent, sessionId: c.sessionId, planId: c.planId })), + columns: [ + { key: 'agent', name: messages.getMessage('output.tableHeader.agent') }, + { key: 'sessionId', name: messages.getMessage('output.tableHeader.sessionId') }, + { key: 'planId', name: messages.getMessage('output.tableHeader.planId') }, + ], + }); + + const confirmed = await yesNoOrCancel({ + message: messages.getMessage('prompt.confirm', [candidates.length]), + default: false, + }); + + if (confirmed === 'cancel' || confirmed === false) { + this.log(messages.getMessage('output.cancelled')); + return []; + } + } + + await Promise.all(candidates.map((c) => unlink(c.path))); + this.log(messages.getMessage('output.deleted', [candidates.length])); + return candidates; + } +} diff --git a/src/commands/agent/trace/list.ts b/src/commands/agent/trace/list.ts index 6768e23a..c3130b7d 100644 --- a/src/commands/agent/trace/list.ts +++ b/src/commands/agent/trace/list.ts @@ -16,7 +16,8 @@ import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core'; import { Messages, SfError } from '@salesforce/core'; -import { listCachedPreviewSessions, listSessionTraces, type TraceFileInfo } from '@salesforce/agents'; +import { listSessionTraces, type TraceFileInfo } from '@salesforce/agents'; +import { listAllAgentSessions } from '../../../agentSessionScanner.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.trace.list'); @@ -71,34 +72,30 @@ export default class AgentTraceList extends SfCommand { const { flags } = await this.parse(AgentTraceList); const agentNameFilter = flags.agent?.toLowerCase(); - - const cachedAgents = await listCachedPreviewSessions(this.project!); + const allSessions = await listAllAgentSessions(this.project!); const result: AgentTraceListResult = []; - for (const { agentId, displayName, sessions } of cachedAgents) { - if (agentNameFilter && !displayName?.toLowerCase().includes(agentNameFilter)) continue; - - for (const { sessionId } of sessions) { - if (flags['session-id'] && sessionId !== flags['session-id']) continue; + for (const { agentId, displayName, sessionId } of allSessions) { + if (agentNameFilter && !displayName.toLowerCase().includes(agentNameFilter)) continue; + if (flags['session-id'] && sessionId !== flags['session-id']) continue; - // eslint-disable-next-line no-await-in-loop - let traces: TraceFileInfo[] = await listSessionTraces(agentId, sessionId); + // eslint-disable-next-line no-await-in-loop + let traces: TraceFileInfo[] = await listSessionTraces(agentId, sessionId); - if (flags.since) { - traces = traces.filter((t) => t.mtime >= flags.since!); - } + if (flags.since) { + traces = traces.filter((t) => t.mtime >= flags.since!); + } - for (const t of traces) { - result.push({ - agent: displayName ?? agentId, - sessionId, - planId: t.planId, - path: t.path, - size: t.size, - mtime: t.mtime.toISOString(), - }); - } + for (const t of traces) { + result.push({ + agent: displayName, + sessionId, + planId: t.planId, + path: t.path, + size: t.size, + mtime: t.mtime.toISOString(), + }); } } diff --git a/src/commands/agent/trace/read.ts b/src/commands/agent/trace/read.ts new file mode 100644 index 00000000..9d0605e1 --- /dev/null +++ b/src/commands/agent/trace/read.ts @@ -0,0 +1,434 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Flags, SfCommand } from '@salesforce/sf-plugins-core'; +import { Messages, SfError } from '@salesforce/core'; +import { + listSessionTraces, + readSessionTrace, + readTurnIndex, + type PlannerResponse, + type PlanStep, + type FunctionStep, +} from '@salesforce/agents'; +import { listAllAgentSessions } from '../../../agentSessionScanner.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.trace.read'); + +export const DIMENSIONS = ['actions', 'grounding', 'routing', 'errors'] as const; +export type Dimension = (typeof DIMENSIONS)[number]; + +// FunctionStep in @salesforce/agents doesn't declare the optional errors field that the API returns +type FunctionStepWithErrors = FunctionStep & { + function: FunctionStep['function'] & { + errors?: Array<{ statusCode: string; message: string }>; + }; +}; + +export type TurnSummary = { + turn: number; + planId: string; + topic: string; + userInput: string; + agentResponse: string; + actionsExecuted: string[]; + latencyMs: number; + error: string | null; +}; + +export type ActionsRow = { + dimension: 'actions'; + turn: number; + planId: string; + action: string; + input: string; + output: string; + latencyMs: number; + error: string | null; +}; +export type GroundingRow = { + dimension: 'grounding'; + turn: number; + planId: string; + prompt: string; + response: string; + latencyMs: number; +}; +export type RoutingRow = { + dimension: 'routing'; + turn: number; + planId: string; + fromTopic: string; + toTopic: string; + intent: string; +}; +export type ErrorsRow = { + dimension: 'errors'; + turn: number; + planId: string; + source: string; + errorCode: string; + message: string; +}; +export type DimensionRow = ActionsRow | GroundingRow | RoutingRow | ErrorsRow; + +export type AgentTraceReadResult = { + sessionId: string; + format: 'summary' | 'detail' | 'raw'; + dimension?: Dimension; + turns?: TurnSummary[]; + detail?: DimensionRow[]; + raw?: PlannerResponse[]; +}; + +const isFunctionStep = (s: PlanStep): s is FunctionStep => s.type === 'FunctionStep'; +const asFunctionWithErrors = (s: FunctionStep): FunctionStepWithErrors => s as FunctionStepWithErrors; + +function summarizeTurn(turn: number, planId: string, trace: PlannerResponse): TurnSummary { + const plan = trace.plan; + const userInput = plan.find((s) => s.type === 'UserInputStep'); + const finalResponse = plan.find((s) => s.type === 'PlannerResponseStep'); + const functionSteps = plan.filter(isFunctionStep).map(asFunctionWithErrors); + + const errorStep = functionSteps.find((s) => s.function.errors?.length); + const errorMsg = errorStep?.function.errors?.[0]?.message ?? null; + const totalLatency = functionSteps.reduce((acc, s) => acc + (s.executionLatency ?? 0), 0); + + return { + turn, + planId, + topic: trace.topic, + userInput: userInput?.type === 'UserInputStep' ? userInput.message : '', + agentResponse: finalResponse?.type === 'PlannerResponseStep' ? finalResponse.message : '', + actionsExecuted: functionSteps.map((s) => s.function.name), + latencyMs: totalLatency, + error: errorMsg, + }; +} + +function extractActions(turn: number, planId: string, trace: PlannerResponse): ActionsRow[] { + return trace.plan + .filter(isFunctionStep) + .map(asFunctionWithErrors) + .map((step) => ({ + dimension: 'actions' as const, + turn, + planId, + action: step.function.name, + input: JSON.stringify(step.function.input), + output: JSON.stringify(step.function.output), + latencyMs: step.executionLatency, + error: step.function.errors?.length ? step.function.errors[0].message : null, + })); +} + +function extractGrounding(turn: number, planId: string, trace: PlannerResponse): GroundingRow[] { + return trace.plan + .filter((s): s is Extract => s.type === 'LLMExecutionStep') + .filter((s) => s.promptName.includes('React')) + .map((step) => ({ + dimension: 'grounding' as const, + turn, + planId, + prompt: step.promptName, + response: step.promptResponse.slice(0, 500), + latencyMs: step.executionLatency, + })); +} + +function extractRouting(turn: number, planId: string, trace: PlannerResponse): RoutingRow[] { + const topicStep = trace.plan.find((s) => s.type === 'UpdateTopicStep'); + const eventStep = trace.plan.find((s) => s.type === 'EventStep' && s.eventName === 'topicChangeEvent'); + const fromTopic = eventStep?.type === 'EventStep' ? eventStep.payload.oldTopic : 'null'; + const toTopic = topicStep?.type === 'UpdateTopicStep' ? topicStep.topic : trace.topic; + return [{ dimension: 'routing' as const, turn, planId, fromTopic, toTopic, intent: trace.intent }]; +} + +function extractErrors(turn: number, planId: string, trace: PlannerResponse): ErrorsRow[] { + const rows: ErrorsRow[] = []; + for (const step of trace.plan) { + if (step.type === 'FunctionStep') { + const errors = asFunctionWithErrors(step).function.errors ?? []; + for (const e of errors) { + rows.push({ + dimension: 'errors', + turn, + planId, + source: step.function.name, + errorCode: e.statusCode, + message: e.message, + }); + } + } + if (step.type === 'EventStep' && step.isError) { + rows.push({ + dimension: 'errors', + turn, + planId, + source: step.eventName, + errorCode: 'EVENT_ERROR', + message: JSON.stringify(step.payload), + }); + } + } + return rows; +} + +async function resolvePlanIds( + agentId: string, + sessionId: string, + turn: number | undefined +): Promise> { + const turnIndex = await readTurnIndex(agentId, sessionId); + + if (turn !== undefined) { + // Try the turn index first (planId may be null if trace wasn't correlated) + const entry = turnIndex?.turns.find((t) => t.turn === turn && t.planId); + if (entry?.planId) { + return [{ turn: entry.turn, planId: entry.planId }]; + } + + // Fall back to positional order from trace files on disk + const traceFiles = await listSessionTraces(agentId, sessionId); + const byPosition = traceFiles[turn - 1]; + if (!byPosition) { + throw new SfError(messages.getMessage('error.turnNotFound', [turn, sessionId]), 'TurnNotFound'); + } + return [{ turn, planId: byPosition.planId }]; + } + + if (turnIndex) { + return turnIndex.turns.filter((t) => t.planId !== null).map((t) => ({ turn: t.turn, planId: t.planId! })); + } + + // Fall back to listing trace files when no turn index exists + const traceFiles = await listSessionTraces(agentId, sessionId); + return traceFiles.map((f, i) => ({ turn: i + 1, planId: f.planId })); +} + +async function readTraces( + agentId: string, + sessionId: string, + planIds: Array<{ turn: number; planId: string }> +): Promise<{ traces: Array<{ turn: number; planId: string; trace: PlannerResponse }>; failedFiles: string[] }> { + const traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> = []; + const failedFiles: string[] = []; + + for (const { turn, planId } of planIds) { + // eslint-disable-next-line no-await-in-loop + const trace = await readSessionTrace(agentId, sessionId, planId); + if (!trace?.plan || !Array.isArray(trace.plan)) { + failedFiles.push(planId); + continue; + } + traces.push({ turn, planId, trace }); + } + + return { traces, failedFiles }; +} + +function extractDimension(turn: number, planId: string, trace: PlannerResponse, dimension: Dimension): DimensionRow[] { + if (dimension === 'actions') return extractActions(turn, planId, trace); + if (dimension === 'grounding') return extractGrounding(turn, planId, trace); + if (dimension === 'routing') return extractRouting(turn, planId, trace); + return extractErrors(turn, planId, trace); +} + +export default class AgentTraceRead extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly flags = { + 'session-id': Flags.string({ + summary: messages.getMessage('flags.session-id.summary'), + required: true, + char: 's', + }), + format: Flags.option({ + options: ['summary', 'detail', 'raw'] as const, + default: 'summary' as const, + summary: messages.getMessage('flags.format.summary'), + char: 'f', + })(), + dimension: Flags.option({ + options: DIMENSIONS, + summary: messages.getMessage('flags.dimension.summary'), + char: 'd', + })(), + turn: Flags.integer({ + summary: messages.getMessage('flags.turn.summary'), + char: 't', + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentTraceRead); + const sessionId = flags['session-id']; + + if (flags.format === 'detail' && !flags.dimension) { + throw new SfError(messages.getMessage('error.detailRequiresDimension'), 'MissingDimension'); + } + if (flags.dimension && flags.format !== 'detail') { + this.warn(messages.getMessage('warn.dimensionIgnored', [flags.format])); + } + + const agentId = await this.resolveAgentId(sessionId); + const planIds = await resolvePlanIds(agentId, sessionId, flags.turn); + + if (planIds.length === 0) { + this.log(messages.getMessage('output.empty')); + return { sessionId, format: flags.format, turns: [], detail: [], raw: [] }; + } + + const { traces, failedFiles } = await readTraces(agentId, sessionId, planIds); + + if (failedFiles.length > 0 && traces.length === 0) { + throw new SfError(messages.getMessage('error.parseFailedAll', [failedFiles.join(', ')]), 'TraceParseError'); + } + if (failedFiles.length > 0) { + this.warn(messages.getMessage('warn.parseFailed', [failedFiles.join(', ')])); + } + + return this.formatOutput(sessionId, flags.format, flags.dimension, traces); + } + + private async resolveAgentId(sessionId: string): Promise { + const allSessions = await listAllAgentSessions(this.project!); + const entry = allSessions.find((s) => s.sessionId === sessionId); + if (!entry) { + throw new SfError(messages.getMessage('error.sessionNotFound', [sessionId]), 'SessionNotFound'); + } + return entry.agentId; + } + + private formatOutput( + sessionId: string, + format: 'summary' | 'detail' | 'raw', + dimension: Dimension | undefined, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + if (format === 'raw') { + const raw = traces.map((t) => t.trace); + if (!this.jsonEnabled()) this.log(JSON.stringify(raw, null, 2)); + return { sessionId, format: 'raw', raw }; + } + + if (format === 'detail') { + return this.formatDetail(sessionId, dimension!, traces); + } + + return this.formatSummary(sessionId, traces); + } + + private formatDetail( + sessionId: string, + dimension: Dimension, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + const detail: DimensionRow[] = traces.flatMap(({ turn, planId, trace }) => + extractDimension(turn, planId, trace, dimension) + ); + + if (detail.length === 0) { + this.log(messages.getMessage('output.emptyDimension', [dimension])); + return { sessionId, format: 'detail', dimension, detail: [] }; + } + + if (!this.jsonEnabled()) { + this.renderDetailTable(dimension, detail); + } + + return { sessionId, format: 'detail', dimension, detail }; + } + + private renderDetailTable(dimension: Dimension, detail: DimensionRow[]): void { + if (dimension === 'actions') { + this.table({ + data: detail as ActionsRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'action', name: messages.getMessage('output.tableHeader.action') }, + { key: 'input', name: messages.getMessage('output.tableHeader.input') }, + { key: 'output', name: messages.getMessage('output.tableHeader.output') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + { key: 'error', name: messages.getMessage('output.tableHeader.error') }, + ], + }); + } else if (dimension === 'grounding') { + this.table({ + data: detail as GroundingRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'prompt', name: messages.getMessage('output.tableHeader.prompt') }, + { key: 'response', name: messages.getMessage('output.tableHeader.response') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + ], + }); + } else if (dimension === 'routing') { + this.table({ + data: detail as RoutingRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'intent', name: messages.getMessage('output.tableHeader.intent') }, + { key: 'fromTopic', name: messages.getMessage('output.tableHeader.fromTopic') }, + { key: 'toTopic', name: messages.getMessage('output.tableHeader.toTopic') }, + ], + }); + } else { + this.table({ + data: detail as ErrorsRow[], + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'source', name: messages.getMessage('output.tableHeader.source') }, + { key: 'errorCode', name: messages.getMessage('output.tableHeader.errorCode') }, + { key: 'message', name: messages.getMessage('output.tableHeader.message') }, + ], + }); + } + } + + private formatSummary( + sessionId: string, + traces: Array<{ turn: number; planId: string; trace: PlannerResponse }> + ): AgentTraceReadResult { + const turns: TurnSummary[] = traces.map(({ turn, planId, trace }) => summarizeTurn(turn, planId, trace)); + + if (!this.jsonEnabled()) { + this.table({ + data: turns.map((t) => ({ + ...t, + actionsExecuted: t.actionsExecuted.join(', ') || '—', + error: t.error ?? '—', + latencyMs: `${t.latencyMs}ms`, + })), + columns: [ + { key: 'turn', name: messages.getMessage('output.tableHeader.turn') }, + { key: 'topic', name: messages.getMessage('output.tableHeader.topic') }, + { key: 'userInput', name: messages.getMessage('output.tableHeader.userInput') }, + { key: 'agentResponse', name: messages.getMessage('output.tableHeader.agentResponse') }, + { key: 'actionsExecuted', name: messages.getMessage('output.tableHeader.actionsExecuted') }, + { key: 'latencyMs', name: messages.getMessage('output.tableHeader.latencyMs') }, + { key: 'error', name: messages.getMessage('output.tableHeader.error') }, + ], + }); + } + + return { sessionId, format: 'summary', turns }; + } +} diff --git a/test/commands/agent/trace/delete.test.ts b/test/commands/agent/trace/delete.test.ts new file mode 100644 index 00000000..53e456a9 --- /dev/null +++ b/test/commands/agent/trace/delete.test.ts @@ -0,0 +1,243 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return */ + +import { join } from 'node:path'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { TestContext } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; + +const MOCK_PROJECT_DIR = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template'); + +// Dates relative to now so --older-than arithmetic stays correct over time. +// RECENT_MTIME: 7 days ago — older than 1h but newer than 28d/30d +// OLD_MTIME: 60 days ago — older than all thresholds used in tests +const RECENT_MTIME = new Date(Date.now() - 7 * 86_400_000); +const OLD_MTIME = new Date(Date.now() - 60 * 86_400_000); + +const MOCK_TRACES_AGENT_A = [ + { planId: 'plan-1', path: '/sfdx/agents/AgentA/sessions/sess-1/traces/plan-1.json', size: 1000, mtime: RECENT_MTIME }, + { planId: 'plan-2', path: '/sfdx/agents/AgentA/sessions/sess-1/traces/plan-2.json', size: 2000, mtime: OLD_MTIME }, +]; +const MOCK_TRACES_AGENT_B = [ + { planId: 'plan-3', path: '/sfdx/agents/AgentB/sessions/sess-2/traces/plan-3.json', size: 3000, mtime: OLD_MTIME }, +]; + +const MOCK_CACHED_SESSIONS = [ + { + agentId: 'AgentA', + displayName: 'My_Agent_A', + sessions: [{ sessionId: 'sess-1', timestamp: RECENT_MTIME.toISOString() }], + }, + { + agentId: 'AgentB', + displayName: 'My_Agent_B', + sessions: [{ sessionId: 'sess-2', timestamp: OLD_MTIME.toISOString() }], + }, +]; + +const MOCK_ALL_SESSIONS = MOCK_CACHED_SESSIONS.flatMap(({ agentId, displayName, sessions }) => + sessions.map(({ sessionId }) => ({ agentId, displayName, sessionId })) +); + +describe('agent trace delete', () => { + const $$ = new TestContext(); + let unlinkStub: sinon.SinonStub; + let listAllAgentSessionsStub: sinon.SinonStub; + let listSessionTracesStub: sinon.SinonStub; + let yesNoOrCancelStub: sinon.SinonStub; + let AgentTraceDelete: any; + + beforeEach(async () => { + unlinkStub = $$.SANDBOX.stub().resolves(); + listAllAgentSessionsStub = $$.SANDBOX.stub().resolves(MOCK_ALL_SESSIONS); + listSessionTracesStub = $$.SANDBOX.stub(); + listSessionTracesStub.withArgs('AgentA', 'sess-1').resolves(MOCK_TRACES_AGENT_A); + listSessionTracesStub.withArgs('AgentB', 'sess-2').resolves(MOCK_TRACES_AGENT_B); + yesNoOrCancelStub = $$.SANDBOX.stub().resolves(true); + + const mod = await esmock('../../../../src/commands/agent/trace/delete.js', { + 'node:fs/promises': { unlink: unlinkStub }, + '../../../../src/agentSessionScanner.js': { + listAllAgentSessions: listAllAgentSessionsStub, + }, + '@salesforce/agents': { + listSessionTraces: listSessionTracesStub, + }, + '../../../../src/yes-no-cancel.js': { default: yesNoOrCancelStub }, + }); + + AgentTraceDelete = mod.default; + + $$.inProject(true); + const mockProject = { getPath: () => MOCK_PROJECT_DIR } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + }); + + describe('with no filters', () => { + it('deletes all traces across all agents when --no-prompt is set', async () => { + const result = await AgentTraceDelete.run(['--no-prompt']); + expect(result).to.have.length(3); + expect(unlinkStub.callCount).to.equal(3); + }); + + it('prompts for confirmation by default', async () => { + await AgentTraceDelete.run([]); + expect(yesNoOrCancelStub.calledOnce).to.be.true; + }); + + it('does not delete when user declines confirmation', async () => { + yesNoOrCancelStub.resolves(false); + const result = await AgentTraceDelete.run([]); + expect(result).to.deep.equal([]); + expect(unlinkStub.called).to.be.false; + }); + + it('does not delete when user cancels confirmation', async () => { + yesNoOrCancelStub.resolves('cancel'); + const result = await AgentTraceDelete.run([]); + expect(result).to.deep.equal([]); + expect(unlinkStub.called).to.be.false; + }); + + it('returns empty and does not prompt when no traces exist', async () => { + listSessionTracesStub.withArgs('AgentA', 'sess-1').resolves([]); + listSessionTracesStub.withArgs('AgentB', 'sess-2').resolves([]); + const result = await AgentTraceDelete.run([]); + expect(result).to.deep.equal([]); + expect(yesNoOrCancelStub.called).to.be.false; + expect(unlinkStub.called).to.be.false; + }); + }); + + describe('--no-prompt', () => { + it('skips the confirmation prompt', async () => { + await AgentTraceDelete.run(['--no-prompt']); + expect(yesNoOrCancelStub.called).to.be.false; + }); + }); + + describe('--agent filter', () => { + it('deletes only traces for the matching agent', async () => { + const result = await AgentTraceDelete.run(['--agent', 'My_Agent_A', '--no-prompt']); + expect(result).to.have.length(2); + expect(result.every((r: any) => r.agent === 'My_Agent_A')).to.be.true; + expect(unlinkStub.callCount).to.equal(2); + }); + + it('uses case-insensitive substring match', async () => { + const result = await AgentTraceDelete.run(['--agent', 'agent_a', '--no-prompt']); + expect(result).to.have.length(2); + }); + + it('returns empty when no agents match', async () => { + const result = await AgentTraceDelete.run(['--agent', 'NonExistent', '--no-prompt']); + expect(result).to.deep.equal([]); + expect(unlinkStub.called).to.be.false; + }); + }); + + describe('--session-id filter', () => { + it('deletes only traces for the specified session', async () => { + const result = await AgentTraceDelete.run(['--session-id', 'sess-2', '--no-prompt']); + expect(result).to.have.length(1); + expect(result[0].sessionId).to.equal('sess-2'); + }); + + it('returns empty when session ID does not match', async () => { + const result = await AgentTraceDelete.run(['--session-id', 'no-such-session', '--no-prompt']); + expect(result).to.deep.equal([]); + }); + }); + + describe('--older-than filter', () => { + it('deletes only traces older than the duration (days)', async () => { + const result = await AgentTraceDelete.run(['--older-than', '30d', '--no-prompt']); + const planIds = result.map((r: any) => r.planId); + expect(planIds).to.include('plan-2'); + expect(planIds).to.include('plan-3'); + expect(planIds).to.not.include('plan-1'); + }); + + it('deletes nothing when all traces are newer than the duration', async () => { + const futureMtime = new Date(Date.now() + 86_400_000); + listSessionTracesStub.withArgs('AgentA', 'sess-1').resolves([ + { + planId: 'plan-1', + path: '/sfdx/agents/AgentA/sessions/sess-1/traces/plan-1.json', + size: 1000, + mtime: futureMtime, + }, + ]); + listSessionTracesStub.withArgs('AgentB', 'sess-2').resolves([]); + const result = await AgentTraceDelete.run(['--older-than', '1d', '--no-prompt']); + expect(result).to.deep.equal([]); + }); + + it('accepts hours unit', async () => { + const result = await AgentTraceDelete.run(['--older-than', '1h', '--no-prompt']); + expect(result).to.have.length(3); + }); + + it('accepts weeks unit', async () => { + const result = await AgentTraceDelete.run(['--older-than', '4w', '--no-prompt']); + const planIds = result.map((r: any) => r.planId); + expect(planIds).to.include('plan-2'); + expect(planIds).to.include('plan-3'); + expect(planIds).to.not.include('plan-1'); + }); + + it('rejects a value without a unit', async () => { + try { + await AgentTraceDelete.run(['--older-than', '7', '--no-prompt']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/invalid.*older-than|InvalidDuration/i); + } + }); + + it('rejects a non-numeric value', async () => { + try { + await AgentTraceDelete.run(['--older-than', 'lastweek', '--no-prompt']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/invalid.*older-than|InvalidDuration/i); + } + }); + }); + + describe('combined filters', () => { + it('applies --agent and --older-than together', async () => { + const result = await AgentTraceDelete.run(['--agent', 'My_Agent_A', '--older-than', '30d', '--no-prompt']); + expect(result).to.have.length(1); + expect(result[0].planId).to.equal('plan-2'); + }); + + it('applies --session-id and --agent together', async () => { + const result = await AgentTraceDelete.run(['--agent', 'My_Agent_A', '--session-id', 'sess-1', '--no-prompt']); + expect(result).to.have.length(2); + expect(result.every((r: any) => r.sessionId === 'sess-1')).to.be.true; + }); + }); +}); diff --git a/test/commands/agent/trace/list.test.ts b/test/commands/agent/trace/list.test.ts index 78e35c6d..00a7ab71 100644 --- a/test/commands/agent/trace/list.test.ts +++ b/test/commands/agent/trace/list.test.ts @@ -25,10 +25,8 @@ import { SfProject } from '@salesforce/core'; const MOCK_PROJECT_DIR = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template'); -// RECENT_MTIME: ~23 days ago from 2026-04-30 -// OLD_MTIME: ~60 days ago from 2026-04-30 -const RECENT_MTIME = new Date('2026-04-07T17:00:00.000Z'); -const OLD_MTIME = new Date('2026-03-01T00:00:00.000Z'); +const RECENT_MTIME = new Date(Date.now() - 7 * 86_400_000); +const OLD_MTIME = new Date(Date.now() - 60 * 86_400_000); const MOCK_TRACES_AGENT_A = [ { planId: 'plan-1', path: '/sfdx/agents/AgentA/sessions/sess-1/traces/plan-1.json', size: 1000, mtime: RECENT_MTIME }, @@ -51,21 +49,27 @@ const MOCK_CACHED_SESSIONS = [ }, ]; +const MOCK_ALL_SESSIONS = MOCK_CACHED_SESSIONS.flatMap(({ agentId, displayName, sessions }) => + sessions.map(({ sessionId }) => ({ agentId, displayName, sessionId })) +); + describe('agent trace list', () => { const $$ = new TestContext(); - let listCachedPreviewSessionsStub: sinon.SinonStub; + let listAllAgentSessionsStub: sinon.SinonStub; let listSessionTracesStub: sinon.SinonStub; let AgentTraceList: any; beforeEach(async () => { - listCachedPreviewSessionsStub = $$.SANDBOX.stub().resolves(MOCK_CACHED_SESSIONS); + listAllAgentSessionsStub = $$.SANDBOX.stub().resolves(MOCK_ALL_SESSIONS); listSessionTracesStub = $$.SANDBOX.stub(); listSessionTracesStub.withArgs('AgentA', 'sess-1').resolves(MOCK_TRACES_AGENT_A); listSessionTracesStub.withArgs('AgentB', 'sess-2').resolves(MOCK_TRACES_AGENT_B); const mod = await esmock('../../../../src/commands/agent/trace/list.js', { + '../../../../src/agentSessionScanner.js': { + listAllAgentSessions: listAllAgentSessionsStub, + }, '@salesforce/agents': { - listCachedPreviewSessions: listCachedPreviewSessionsStub, listSessionTraces: listSessionTracesStub, }, }); @@ -102,7 +106,7 @@ describe('agent trace list', () => { }); it('returns empty when no sessions exist', async () => { - listCachedPreviewSessionsStub.resolves([]); + listAllAgentSessionsStub.resolves([]); const result = await AgentTraceList.run([]); expect(result).to.deep.equal([]); }); diff --git a/test/commands/agent/trace/read.test.ts b/test/commands/agent/trace/read.test.ts new file mode 100644 index 00000000..0f29e3d5 --- /dev/null +++ b/test/commands/agent/trace/read.test.ts @@ -0,0 +1,468 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unnecessary-type-assertion */ + +import { join } from 'node:path'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import esmock from 'esmock'; +import { TestContext } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; + +const MOCK_PROJECT_DIR = join(process.cwd(), 'test', 'mock-projects', 'agent-generate-template'); + +const SESSION_ID = 'sess-abc'; +const AGENT_ID = 'AgentA'; +const PLAN_ID_1 = 'plan-1'; +const PLAN_ID_2 = 'plan-2'; + +const MOCK_CACHED_SESSIONS = [ + { + agentId: AGENT_ID, + displayName: 'My_Agent_A', + sessions: [{ sessionId: SESSION_ID, timestamp: '2026-04-07T17:00:00.000Z' }], + }, +]; + +const MOCK_ALL_SESSIONS = MOCK_CACHED_SESSIONS.flatMap(({ agentId, displayName, sessions }) => + sessions.map(({ sessionId }) => ({ agentId, displayName, sessionId })) +); + +const MOCK_TURN_INDEX = { + version: '1', + sessionId: SESSION_ID, + agentId: AGENT_ID, + created: '2026-04-07T17:00:00.000Z', + turns: [ + { + turn: 1, + timestamp: '2026-04-07T17:00:00.000Z', + role: 'user', + summary: 'Hi!', + summaryTruncated: false, + multiModal: null, + traceFile: `traces/${PLAN_ID_1}.json`, + planId: PLAN_ID_1, + }, + { + turn: 2, + timestamp: '2026-04-07T17:00:01.000Z', + role: 'user', + summary: "what's the weather", + summaryTruncated: false, + multiModal: null, + traceFile: `traces/${PLAN_ID_2}.json`, + planId: PLAN_ID_2, + }, + ], +}; + +// Simple off-topic trace (no actions, no errors) +const MOCK_TRACE_1 = { + type: 'PlanSuccessResponse', + planId: PLAN_ID_1, + sessionId: SESSION_ID, + intent: 'Off_Topic', + topic: 'Off_Topic', + plan: [ + { type: 'UserInputStep', message: 'Hi!' }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactTopicPrompt', + promptContent: 'classify...', + promptResponse: 'Off_Topic', + executionLatency: 460, + startExecutionTime: 1000, + endExecutionTime: 1460, + }, + { + type: 'UpdateTopicStep', + topic: 'Off_Topic', + description: 'Off topic', + job: 'redirect', + instructions: [], + availableFunctions: [], + }, + { + type: 'EventStep', + eventName: 'topicChangeEvent', + isError: false, + payload: { oldTopic: 'null', newTopic: 'Off_Topic' }, + }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactInitialPrompt', + promptContent: 'system...', + promptResponse: 'Hey there!', + executionLatency: 1637, + startExecutionTime: 1461, + endExecutionTime: 3098, + }, + { + type: 'PlannerResponseStep', + message: 'Hey there! How can I assist you today?', + responseType: 'Inform', + isContentSafe: true, + }, + ], +}; + +// Weather trace with action + error +const MOCK_TRACE_2 = { + type: 'PlanSuccessResponse', + planId: PLAN_ID_2, + sessionId: SESSION_ID, + intent: 'Local_Weather', + topic: 'Local_Weather', + plan: [ + { type: 'UserInputStep', message: "what's the weather" }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactTopicPrompt', + promptContent: 'classify...', + promptResponse: 'Local_Weather', + executionLatency: 572, + startExecutionTime: 2000, + endExecutionTime: 2572, + }, + { + type: 'UpdateTopicStep', + topic: 'Local_Weather', + description: 'Weather', + job: 'answer weather questions', + instructions: [], + availableFunctions: ['Check_Weather'], + }, + { + type: 'LLMExecutionStep', + promptName: 'AiCopilot__ReactInitialPrompt', + promptContent: 'system...', + promptResponse: + '- id: call_xxx\n function:\n name: Check_Weather\n arguments: \'{"dateToCheck":"2025-08-18"}\'', + executionLatency: 748, + startExecutionTime: 2600, + endExecutionTime: 3348, + }, + { + type: 'FunctionStep', + function: { + name: 'Check_Weather', + input: { dateToCheck: '2025-08-18' }, + output: {}, + errors: [{ statusCode: 'UNKNOWN_EXCEPTION', message: 'Bad response: 404' }], + }, + executionLatency: 781, + startExecutionTime: 3350, + endExecutionTime: 4131, + }, + { + type: 'PlannerResponseStep', + message: "I'm having trouble accessing the weather.", + responseType: 'Inform', + isContentSafe: true, + }, + ], +}; + +describe('agent trace read', () => { + const $$ = new TestContext(); + let listAllAgentSessionsStub: sinon.SinonStub; + let listSessionTracesStub: sinon.SinonStub; + let readSessionTraceStub: sinon.SinonStub; + let readTurnIndexStub: sinon.SinonStub; + let AgentTraceRead: any; + + beforeEach(async () => { + listAllAgentSessionsStub = $$.SANDBOX.stub().resolves(MOCK_ALL_SESSIONS); + listSessionTracesStub = $$.SANDBOX.stub().resolves([]); + readTurnIndexStub = $$.SANDBOX.stub().resolves(MOCK_TURN_INDEX); + readSessionTraceStub = $$.SANDBOX.stub(); + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_1).resolves(MOCK_TRACE_1); + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_2); + + const mod = await esmock('../../../../src/commands/agent/trace/read.js', { + '../../../../src/agentSessionScanner.js': { + listAllAgentSessions: listAllAgentSessionsStub, + }, + '@salesforce/agents': { + listSessionTraces: listSessionTracesStub, + readSessionTrace: readSessionTraceStub, + readTurnIndex: readTurnIndexStub, + }, + }); + + AgentTraceRead = mod.default; + + $$.inProject(true); + const mockProject = { getPath: () => MOCK_PROJECT_DIR } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + }); + + describe('--format summary (default)', () => { + it('returns summary for all turns when no --turn specified', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.format).to.equal('summary'); + expect(result.turns).to.have.length(2); + }); + + it('each turn has required fields', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + const turn = result.turns[0]; + expect(turn).to.have.keys([ + 'turn', + 'planId', + 'topic', + 'userInput', + 'agentResponse', + 'actionsExecuted', + 'latencyMs', + 'error', + ]); + }); + + it('populates topic and userInput from trace', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[0].topic).to.equal('Off_Topic'); + expect(result.turns[0].userInput).to.equal('Hi!'); + }); + + it('lists executed actions', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[1].actionsExecuted).to.deep.equal(['Check_Weather']); + }); + + it('captures error from failed function step', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[1].error).to.equal('Bad response: 404'); + }); + + it('error is null when no function errors exist', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns[0].error).to.be.null; + }); + + it('scopes to a single turn with --turn', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '1']); + expect(result.turns).to.have.length(1); + expect(result.turns![0].turn).to.equal(1); + }); + }); + + describe('--format detail', () => { + it('throws when --dimension is missing', async () => { + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'detail']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('--dimension'); + } + }); + + describe('--dimension actions', () => { + it('returns only action rows', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.format).to.equal('detail'); + expect(result.dimension).to.equal('actions'); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ action: 'Check_Weather' }); + }); + + it('includes error details in action row', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.detail![0].error).to.equal('Bad response: 404'); + }); + + it('returns empty when no actions exist', async () => { + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_1); + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'actions', + ]); + expect(result.detail).to.deep.equal([]); + }); + }); + + describe('--dimension routing', () => { + it('returns routing rows for each turn', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'routing', + ]); + expect(result.detail).to.have.length(2); + expect(result.detail![0]).to.include({ fromTopic: 'null', toTopic: 'Off_Topic', intent: 'Off_Topic' }); + }); + + it('scopes to a single turn with --turn', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'routing', + '--turn', + '2', + ]); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ intent: 'Local_Weather' }); + }); + }); + + describe('--dimension errors', () => { + it('returns rows only for turns with errors', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'errors', + ]); + expect(result.detail).to.have.length(1); + expect(result.detail![0]).to.include({ source: 'Check_Weather', errorCode: 'UNKNOWN_EXCEPTION' }); + }); + + it('returns empty when no errors exist', async () => { + readSessionTraceStub.withArgs(AGENT_ID, SESSION_ID, PLAN_ID_2).resolves(MOCK_TRACE_1); + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'errors', + ]); + expect(result.detail).to.deep.equal([]); + }); + }); + + describe('--dimension grounding', () => { + it('returns LLM execution steps with React prompts', async () => { + const result = await AgentTraceRead.run([ + '--session-id', + SESSION_ID, + '--format', + 'detail', + '--dimension', + 'grounding', + ]); + expect(result.detail!.length).to.be.greaterThan(0); + for (const row of result.detail!) { + expect((row as any).prompt).to.include('React'); + } + }); + }); + }); + + describe('--format raw', () => { + it('returns raw trace objects', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'raw']); + expect(result.format).to.equal('raw'); + expect(result.raw).to.have.length(2); + expect(result.raw![0].planId).to.equal(PLAN_ID_1); + }); + + it('raw output matches the full trace structure', async () => { + const result = await AgentTraceRead.run(['--session-id', SESSION_ID, '--format', 'raw']); + expect(result.raw![0]).to.deep.equal(MOCK_TRACE_1); + }); + }); + + describe('validation and error handling', () => { + it('throws when session is not found', async () => { + listAllAgentSessionsStub.resolves([]); + try { + await AgentTraceRead.run(['--session-id', 'no-such-session']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('no-such-session'); + } + }); + + it('throws when --turn is out of range (no index, no trace files)', async () => { + readTurnIndexStub.resolves(null); + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '1']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/turn 1|not found/i); + } + }); + + it('throws when --turn number does not exist in the index', async () => { + try { + await AgentTraceRead.run(['--session-id', SESSION_ID, '--turn', '99']); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/turn 99|not found/i); + } + }); + + it('throws when all trace files fail to parse', async () => { + readSessionTraceStub.resetBehavior(); + readSessionTraceStub.resolves(null); + try { + await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect.fail('Should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/Trace parsing failed|raw/i); + } + }); + + it('returns empty result when session has no trace files and no turn index', async () => { + readTurnIndexStub.resolves(null); + listSessionTracesStub.resolves([]); + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns).to.deep.equal([]); + }); + + it('falls back to listSessionTraces when no turn index exists', async () => { + readTurnIndexStub.resolves(null); + listSessionTracesStub.resolves([{ planId: PLAN_ID_1, path: '/path/plan-1.json', size: 1000, mtime: new Date() }]); + const result = await AgentTraceRead.run(['--session-id', SESSION_ID]); + expect(result.turns).to.have.length(1); + expect(result.turns![0].topic).to.equal('Off_Topic'); + }); + }); +}); diff --git a/test/nuts/z4.agent.trace.list.nut.ts b/test/nuts/z4.agent.trace.list.nut.ts index a8c255e1..204afe89 100644 --- a/test/nuts/z4.agent.trace.list.nut.ts +++ b/test/nuts/z4.agent.trace.list.nut.ts @@ -37,19 +37,19 @@ describe('agent trace list', function () { const targetOrg = getUsername(); const startResult = execCmd( `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --target-org ${targetOrg} --json`, - { ensureExitCode: 0 } + { ensureExitCode: 0, cwd: session.project.dir } ).jsonOutput?.result; expect(startResult?.sessionId).to.be.a('string'); sessionId = startResult!.sessionId; execCmd( `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "What can you help me with?" --target-org ${targetOrg} --json`, - { ensureExitCode: 0 } + { ensureExitCode: 0, cwd: session.project.dir } ); execCmd( `agent preview end --session-id ${sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, - { ensureExitCode: 0 } + { ensureExitCode: 0, cwd: session.project.dir } ); }); diff --git a/test/nuts/z4.agent.trace.read.nut.ts b/test/nuts/z4.agent.trace.read.nut.ts new file mode 100644 index 00000000..7fafe989 --- /dev/null +++ b/test/nuts/z4.agent.trace.read.nut.ts @@ -0,0 +1,181 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; +import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; +import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; +import type { AgentTraceReadResult } from '../../src/commands/agent/trace/read.js'; +import { getTestSession, getUsername } from './shared-setup.js'; + +describe('agent trace read', function () { + this.timeout(30 * 60 * 1000); + + let session: TestSession; + let sessionId: string; + const bundleApiName = 'Willie_Resort_Manager'; + + before(async function () { + this.timeout(30 * 60 * 1000); + session = await getTestSession(); + + // Start a preview session with two turns so there are traces to read + const targetOrg = getUsername(); + const startResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(startResult?.sessionId).to.be.a('string'); + sessionId = startResult!.sessionId; + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "What can you help me with?" --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "Tell me more" --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + + execCmd( + `agent preview end --session-id ${sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + }); + + describe('--format summary (default)', () => { + it('returns a summary result for the session', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.format).to.equal('summary'); + expect(result?.sessionId).to.equal(sessionId); + expect(result?.turns).to.be.an('array').with.length.greaterThan(0); + }); + + it('each turn has the required summary fields', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + const turn = result!.turns![0]; + expect(turn).to.have.keys([ + 'turn', + 'planId', + 'topic', + 'userInput', + 'agentResponse', + 'actionsExecuted', + 'latencyMs', + 'error', + ]); + expect(turn.userInput).to.be.a('string').and.have.length.greaterThan(0); + expect(turn.agentResponse).to.be.a('string').and.have.length.greaterThan(0); + expect(turn.topic).to.be.a('string').and.have.length.greaterThan(0); + }); + + it('scopes to a single turn with --turn 1', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --turn 1 --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.turns).to.have.length(1); + expect(result?.turns![0].turn).to.equal(1); + }); + }); + + describe('--format detail', () => { + it('errors when --dimension is missing', () => { + execCmd(`agent trace read --session-id ${sessionId} --format detail --json`, { + ensureExitCode: 1, + cwd: session.project.dir, + }); + }); + + it('returns routing dimension rows', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension routing --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.dimension).to.equal('routing'); + expect(result?.detail).to.be.an('array').with.length.greaterThan(0); + const row = result!.detail![0] as { turn: number; intent: string; toTopic: string }; + expect(row.turn).to.be.a('number'); + expect(row.intent).to.be.a('string'); + expect(row.toTopic).to.be.a('string'); + }); + + it('returns grounding dimension rows', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension grounding --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.dimension).to.equal('grounding'); + expect(result?.detail).to.be.an('array'); + if (result!.detail!.length > 0) { + const row = result!.detail![0] as { prompt: string; latencyMs: number }; + expect(row.prompt).to.be.a('string'); + expect(row.latencyMs).to.be.a('number').and.greaterThanOrEqual(0); + } + }); + + it('returns actions dimension rows (may be empty for off-topic sessions)', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension actions --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.detail).to.be.an('array'); + }); + + it('returns errors dimension rows (may be empty for successful sessions)', () => { + const result = execCmd( + `agent trace read --session-id ${sessionId} --format detail --dimension errors --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result?.format).to.equal('detail'); + expect(result?.detail).to.be.an('array'); + }); + }); + + describe('--format raw', () => { + it('returns raw trace JSON', () => { + const result = execCmd(`agent trace read --session-id ${sessionId} --format raw --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result?.format).to.equal('raw'); + expect(result?.raw).to.be.an('array').with.length.greaterThan(0); + const trace = result!.raw![0] as { type: string; plan: unknown[] }; + expect(trace.type).to.equal('PlanSuccessResponse'); + expect(trace.plan).to.be.an('array').with.length.greaterThan(0); + }); + }); + + describe('error handling', () => { + it('errors when session is not found', () => { + execCmd('agent trace read --session-id no-such-session --json', { + ensureExitCode: 1, + cwd: session.project.dir, + }); + }); + }); +}); diff --git a/test/nuts/z5.agent.trace.delete.nut.ts b/test/nuts/z5.agent.trace.delete.nut.ts new file mode 100644 index 00000000..db5cd8c7 --- /dev/null +++ b/test/nuts/z5.agent.trace.delete.nut.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; +import { execCmd, TestSession } from '@salesforce/cli-plugins-testkit'; +import type { AgentPreviewStartResult } from '../../src/commands/agent/preview/start.js'; +import type { AgentPreviewSendResult } from '../../src/commands/agent/preview/send.js'; +import type { AgentPreviewEndResult } from '../../src/commands/agent/preview/end.js'; +import type { AgentTraceDeleteResult } from '../../src/commands/agent/trace/delete.js'; +import { getTestSession, getUsername } from './shared-setup.js'; + +describe('agent trace delete', function () { + this.timeout(30 * 60 * 1000); + + let session: TestSession; + let sessionId: string; + const bundleApiName = 'Willie_Resort_Manager'; + + before(async function () { + this.timeout(30 * 60 * 1000); + session = await getTestSession(); + + // Start a preview session so there are traces to delete + const targetOrg = getUsername(); + const startResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(startResult?.sessionId).to.be.a('string'); + sessionId = startResult!.sessionId; + + execCmd( + `agent preview send --session-id ${sessionId} --authoring-bundle ${bundleApiName} --utterance "What can you help me with?" --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + + execCmd( + `agent preview end --session-id ${sessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + }); + + it('returns empty array when no traces match the filter', () => { + const result = execCmd( + 'agent trace delete --session-id no-such-session --no-prompt --json', + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result).to.deep.equal([]); + }); + + it('deletes traces for a specific session and returns deleted entries', () => { + const result = execCmd(`agent trace delete --session-id ${sessionId} --no-prompt --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result).to.be.an('array').with.length.greaterThan(0); + expect(result!.every((r) => r.sessionId === sessionId)).to.be.true; + }); + + it('each deleted entry has required fields', () => { + // Create a fresh session to delete + const targetOrg = getUsername(); + const startResult = execCmd( + `agent preview start --authoring-bundle ${bundleApiName} --simulate-actions --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + const newSessionId = startResult!.sessionId; + + execCmd( + `agent preview send --session-id ${newSessionId} --authoring-bundle ${bundleApiName} --utterance "Hello" --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + execCmd( + `agent preview end --session-id ${newSessionId} --authoring-bundle ${bundleApiName} --target-org ${targetOrg} --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ); + + const result = execCmd( + `agent trace delete --session-id ${newSessionId} --no-prompt --json`, + { ensureExitCode: 0, cwd: session.project.dir } + ).jsonOutput?.result; + expect(result).to.be.an('array').with.length.greaterThan(0); + const entry = result![0]; + expect(entry).to.have.keys(['agent', 'sessionId', 'planId', 'path']); + expect(entry.sessionId).to.equal(newSessionId); + }); + + it('deletes traces older than a given duration with --older-than', () => { + // All traces just created are only seconds old, so --older-than 1d should delete nothing + const result = execCmd('agent trace delete --older-than 1d --no-prompt --json', { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + // Traces just created should not match --older-than 1d + expect(result).to.be.an('array'); + }); + + it('deletes all remaining traces with --no-prompt (cleanup)', () => { + const result = execCmd(`agent trace delete --agent ${bundleApiName} --no-prompt --json`, { + ensureExitCode: 0, + cwd: session.project.dir, + }).jsonOutput?.result; + expect(result).to.be.an('array'); + }); +}); diff --git a/yarn.lock b/yarn.lock index c821e3d4..0895b3de 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1792,7 +1792,7 @@ cli-progress "^3.12.0" terminal-link "^3.0.0" -"@salesforce/source-deploy-retrieve@^12.32.8", "@salesforce/source-deploy-retrieve@^12.35.1": +"@salesforce/source-deploy-retrieve@^12.35.1", "@salesforce/source-deploy-retrieve@^12.35.3": version "12.35.4" resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.35.4.tgz#e2b4dc2270a1759f26c13ecf390ad37be03d4c94" integrity sha512-Wuz+qD11ek6DfHNk2gH7shfxjjk98nSGRh/0kY5a4dJz2lslDJIHFIiMoocT7O1Wl0i6qAS85NOek9Z3xWteGw==