Push agents to act when they end a turn announcing work

A local model often ends a turn with "let me set that up now" and no tool
call. Nothing runs after a turn ends, so the user waits on work that never
happens. A text-only turn that announces unfinished work now gets up to two
follow-ups that ask the agent to do it or report what is open.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011VfFvSskgrByEU75QLyreE
This commit is contained in:
Claude 2026-10-22 17:18:20 +00:00
parent 9a753e9adb
commit aa989f50b3
No known key found for this signature in database
4 changed files with 205 additions and 0 deletions

View File

@ -0,0 +1,36 @@
import { describe, expect, it } from "vitest";
import { announcesUnfinishedWork } from "./announced-work.js";
describe("announcesUnfinishedWork", () => {
it.each([
"Nein, noch kein separates Repository. Lass mich das jetzt anlegen.",
"Fertig: Vite+React, 6 Sektionen.\n\nNoch offen: Deployment. Ich arbeite gerade am Build und melde mich.",
"Dependencies installiert. Jetzt bauen und testen:",
"Ich lege jetzt das Repo an.",
"Jetzt baue ich die Seite.",
"The domain is free. I'll now deploy the page.",
"Let me check the server logs first.",
"I'm working on the build right now.",
])("flags %j", (text) => {
expect(announcesUnfinishedWork(text)).toBe(true);
});
it.each([
"Die Landingpage ist live unter http://example.test:8082/ und funktioniert.",
"Hier sind alle drei Logo-Varianten.\n\nLass mich wissen, welche dir gefällt.",
"Done. Let me know if you want changes.",
"Ich starte jetzt einen neuen Task beim Grafiker. Welchen Stil möchtest du?",
"Sag mir Bescheid, wenn ich weitermachen soll.",
"",
])("accepts %j", (text) => {
expect(announcesUnfinishedWork(text)).toBe(false);
});
it("only looks at the closing paragraph", () => {
expect(
announcesUnfinishedWork(
"Lass mich kurz zusammenfassen.\n\nDie Seite ist gebaut, gepusht und live.",
),
).toBe(false);
});
});

View File

@ -0,0 +1,51 @@
/**
* Local models often end a turn by announcing the next step ("Lass mich das jetzt anlegen",
* "I'll now deploy it") without calling a tool. Nothing runs after a turn ends, so the user is
* left waiting on work that never happens. This spots such endings so the runtime can push the
* agent once more to either do the work or report honestly.
*/
/** Phrases that look like announcements but hand the turn to the user. */
const HANDS_BACK =
/\b(?:lass(?:t)? (?:mich|uns) (?:es )?wissen|sag(?:t)? (?:mir )?bescheid|gib (?:mir )?bescheid|let me know|tell me)\b/gi;
const ANNOUNCES_WORK = [
// German: "Lass mich …", "Ich lege jetzt …", "Jetzt baue ich …", "Ich arbeite gerade am Build".
/\blass(?:t)? (?:mich|uns)\b/i,
/\bich (?:werde|starte|mache|baue|lege|erstelle|prüfe|kümmere mich|schaue|deploye|beginne|fange|schreibe|richte|lade|teste)\b[^.!?\n]*\b(?:jetzt|gleich|nun|sofort|als nächstes|direkt)\b/i,
/\b(?:jetzt|nun|gleich|als nächstes)\s+(?:lege|baue|erstelle|mache|starte|deploye|prüfe|schreibe|kümmere|richte|teste|lade)\s+ich\b/i,
/\bich arbeite (?:gerade|jetzt|noch|daran)\b/i,
/\bbin (?:gerade |jetzt )?dabei\b/i,
// English.
/\blet me\b/i,
/\bi(?:'ll| will) (?:now|start|go ahead|get started|create|build|deploy|set up|write|fix|run)\b/i,
/\bnow i(?:'ll| will)\b/i,
/\bnext,? i(?:'ll| will)\b/i,
/\bi(?:'m| am) (?:now |currently |still )?(?:working on|starting|building|creating|setting up|deploying)\b/i,
];
/** Last paragraph of the reply, where a closing announcement sits. */
function closingText(text: string): string {
const paragraphs = text
.trim()
.split(/\n\s*\n/)
.map((part) => part.trim())
.filter(Boolean);
return (paragraphs.at(-1) ?? "").slice(-400);
}
export function announcesUnfinishedWork(text: string): boolean {
const closing = closingText(text);
if (!closing) return false;
// A question hands the turn to the user; waiting for the answer is right.
if (/\?\s*\W*$/u.test(closing)) return false;
// "Jetzt bauen und testen:" — the turn ends where a tool call should follow.
if (/:\s*$/.test(closing)) return true;
const withoutHandBack = closing.replace(HANDS_BACK, "");
return ANNOUNCES_WORK.some((pattern) => pattern.test(withoutHandBack));
}
export const MAX_ANNOUNCED_WORK_CONTINUATIONS = 2;
export const ANNOUNCED_WORK_CONTINUATION_PROMPT =
"Your last message announced work you have not done, and nothing runs after your turn ends. Do that work now with your tools. If you cannot, tell the user plainly what is done, what is still open, and what you need from them.";

View File

@ -7,6 +7,7 @@ const fakeAgentState = vi.hoisted(() => ({
| "empty" | "empty"
| "two-boundaries" | "two-boundaries"
| "silent-continuation" | "silent-continuation"
| "announce"
| "subagent-limit" | "subagent-limit"
| "parent-limit" | "parent-limit"
| "parent-parallel" | "parent-parallel"
@ -134,6 +135,35 @@ vi.mock("@earendil-works/pi-agent-core", () => ({
return; return;
} }
if (fakeAgentState.mode === "announce") {
const announce = (text: string) => {
this.emit({
type: "message_update",
assistantMessageEvent: { type: "text_delta", delta: text },
});
this.emit({
type: "message_end",
message: { role: "assistant", content: [{ type: "text", text }] },
});
this.emit({
type: "turn_end",
message: { role: "assistant", content: [{ type: "text", text }] },
toolResults: [],
});
};
// Keeps announcing; the runtime nudges a bounded number of times, then lets it end.
announce("Lass mich das Repo jetzt anlegen.");
while (fakeAgentState.followUpMessages.length > fakeAgentState.promptInputs.length - 1) {
fakeAgentState.promptInputs.push("follow-up");
announce(
fakeAgentState.emitFinalAfterFollowUp
? "Das Repo ist angelegt."
: "Ich lege das Repo jetzt an.",
);
}
return;
}
if (fakeAgentState.mode === "parent-limit") { if (fakeAgentState.mode === "parent-limit") {
const shell = this.tools.find((tool) => tool.name === "shell"); const shell = this.tools.find((tool) => tool.name === "shell");
if (!shell) throw new Error("shell was not exposed"); if (!shell) throw new Error("shell was not exposed");
@ -731,6 +761,72 @@ describe("Pi connector tool dispatch", () => {
}); });
}); });
it("pushes once more when a text-only turn announces work it did not do", async () => {
fakeAgentState.mode = "announce";
const runtime = new PiAgentRuntime();
const events: unknown[] = [];
for await (const event of runtime.run(
{
botId: "b",
threadId: "t",
runId: "announce",
prompt: "leg ein Repo an",
instructions: "Use your tools.",
history: [],
tools: [destinationTool],
model: { provider: "test", id: "dispatch-test-model" },
executeTool: vi.fn(async () => ({ ok: true })),
},
{
operationId: "announce",
traceId: "announce",
spaceId: "w",
userId: "u",
signal: new AbortController().signal,
},
)) {
events.push(event);
}
expect(fakeAgentState.followUpMessages).toEqual([
expect.objectContaining({
role: "user",
content: expect.stringContaining("nothing runs after your turn ends"),
}),
]);
expect(events).toContainEqual({ type: "text", text: "Das Repo ist angelegt." });
});
it("stops nudging an agent that keeps announcing", async () => {
fakeAgentState.mode = "announce";
fakeAgentState.emitFinalAfterFollowUp = false;
const runtime = new PiAgentRuntime();
for await (const _event of runtime.run(
{
botId: "b",
threadId: "t",
runId: "announce-loop",
prompt: "leg ein Repo an",
instructions: "Use your tools.",
history: [],
tools: [destinationTool],
model: { provider: "test", id: "dispatch-test-model" },
executeTool: vi.fn(async () => ({ ok: true })),
},
{
operationId: "announce-loop",
traceId: "announce-loop",
spaceId: "w",
userId: "u",
signal: new AbortController().signal,
},
)) {
// drain
}
expect(fakeAgentState.followUpMessages).toHaveLength(2);
});
it("does not wait for a slow tool completion hook", async () => { it("does not wait for a slow tool completion hook", async () => {
let resolveAudit!: () => void; let resolveAudit!: () => void;
const audit = new Promise<void>((resolve) => { const audit = new Promise<void>((resolve) => {

View File

@ -12,6 +12,11 @@ import type {
} from "@rakazo/adapter-kit"; } from "@rakazo/adapter-kit";
import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts"; import { MODEL_TIERS, ModelTierSchema } from "@rakazo/contracts";
import { getLogger } from "@rakazo/logging"; import { getLogger } from "@rakazo/logging";
import {
ANNOUNCED_WORK_CONTINUATION_PROMPT,
announcesUnfinishedWork,
MAX_ANNOUNCED_WORK_CONTINUATIONS,
} from "./announced-work.js";
import { isToolPauseResult } from "./approval-effect.js"; import { isToolPauseResult } from "./approval-effect.js";
import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js"; import { DELEGATION_TOOL_NAMES, SUBAGENT_MODEL_TIER_DESCRIPTION } from "./builtin-tools.js";
import { CATALOG_EXECUTE, prepareCatalogExecuteArguments } from "./lazy-tool-catalog.js"; import { CATALOG_EXECUTE, prepareCatalogExecuteArguments } from "./lazy-tool-catalog.js";
@ -341,6 +346,7 @@ export class PiAgentRuntime implements AgentRuntime {
let toolCalls = 0; let toolCalls = 0;
let toolActivityShowing = false; let toolActivityShowing = false;
let silentToolContinuations = 0; let silentToolContinuations = 0;
let announcedWorkContinuations = 0;
let toolWorkPendingFinal = false; let toolWorkPendingFinal = false;
agent.subscribe(async (event) => { agent.subscribe(async (event) => {
if (event.type === "message_end") { if (event.type === "message_end") {
@ -405,6 +411,22 @@ export class PiAgentRuntime implements AgentRuntime {
}); });
} }
} }
// A text-only turn that announces the next step ends the run with nothing done.
if (
!hasToolCalls &&
!hasToolResults &&
!host.pausePending &&
tools.length > 0 &&
announcedWorkContinuations < MAX_ANNOUNCED_WORK_CONTINUATIONS &&
announcesUnfinishedWork(messageText)
) {
announcedWorkContinuations += 1;
agent.followUp({
role: "user",
content: ANNOUNCED_WORK_CONTINUATION_PROMPT,
timestamp: Date.now(),
});
}
} }
if (event.type === "message_end" && event.message.role === "assistant") { if (event.type === "message_end" && event.message.role === "assistant") {
const text = assistantText(event.message); const text = assistantText(event.message);