/** * Background Routine Self-Healing, Scheduling Lifecycle & Quiet Notification Engine. * * Implements Feature F8 per PROJECT.md and ORIGINAL_REQUEST.md. */ import type { RoutineNotifyMode } from "./routine-policy.js"; export interface RoutineRetryConfig { maxAttempts: number; baseBackoffMs: number; maxBackoffMs: number; } export const DEFAULT_ROUTINE_RETRY_CONFIG: RoutineRetryConfig = { maxAttempts: 3, baseBackoffMs: 1_000, maxBackoffMs: 60_000, }; /** * Classifies whether an error encountered during a routine run is transient (retryable) * or terminal (permanent). */ export function isTransientRoutineError(error: unknown): boolean { if (!error) return false; const message = error instanceof Error ? error.message : String(error); const code = (error as { code?: string })?.code; const status = (error as { status?: number; statusCode?: number })?.status ?? (error as { statusCode?: number })?.statusCode; // 1. Explicit terminal signals (fail-fast) if ( status === 401 || status === 403 || /unauthorized|forbidden|invalid api key|invalid token|access denied/i.test(message) ) { return false; } if (/validation failed|invalid arguments|syntax error/i.test(message)) { return false; } if (/budget exceeded|max steps reached|loop guard/i.test(message)) { return false; } // 2. HTTP transient status codes if ( status === 408 || status === 429 || (status !== undefined && status >= 500 && status <= 504) ) { return true; } // 3. Network and socket level transient codes const transientNetworkCodes = new Set([ "ECONNRESET", "ETIMEDOUT", "ECONNREFUSED", "EAI_AGAIN", "ENOTFOUND", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "UND_ERR_HEADERS_TIMEOUT", ]); if (code && transientNetworkCodes.has(code)) { return true; } // 4. Undici, rate-limit, overload, and generic transient HTTP/network error strings if ( /fetch failed|network timeout|socket hang up|connection reset|timed out|rate limit|too many requests|overloaded/i.test( message, ) || /\b(429|500|502|503|504)\b/.test(message) ) { return true; } // 5. Database concurrency & lock contention if ( code === "53300" || code === "P2034" || /too many database connections|deadlock detected/i.test(message) ) { return true; } // 6. Computer / Sandbox temporary busy state if (/computer busy|sandbox unavailable|container booting/i.test(message)) { return true; } return false; } /** * Calculates exponential backoff with full jitter for routine retries. * Prevents thundering herd using decorrelated random jitter. */ export function calculateRoutineBackoffMs( attempt: number, config: Partial = {}, ): number { const cfg = { ...DEFAULT_ROUTINE_RETRY_CONFIG, ...config }; if (attempt <= 0) return 0; // Exponential backoff: base * 2^(attempt - 1) const factor = 2 ** Math.min(attempt - 1, 6); const boundedMax = Math.min(cfg.maxBackoffMs, cfg.baseBackoffMs * factor); // Full Jitter: random value between [0, boundedMax] return Math.floor(Math.random() * boundedMax); } export interface RoutineNotificationEvaluationInput { routineNotifyEnabled: boolean; notifyMode: RoutineNotifyMode; hasChanged: boolean; outcome: "completed" | "failed"; } export interface RoutineRunnerDecision { shouldNotify: boolean; markUnread: boolean; isQuiet: boolean; reason: string; } /** * Evaluates whether a background routine completion or failure should notify the user, * mark the thread unread, or execute quietly without nagging. */ export function evaluateRoutineNotificationPolicy( input: RoutineNotificationEvaluationInput, ): RoutineRunnerDecision { // If routine level notify flag is disabled, all notifications are muted if (!input.routineNotifyEnabled || input.notifyMode === "never") { return { shouldNotify: false, markUnread: false, isQuiet: true, reason: "Routine notifications are disabled", }; } if (input.outcome === "failed") { const notifyOnFailure = input.notifyMode === "always" || input.notifyMode === "on_failure" || input.notifyMode === "on_change"; return { shouldNotify: notifyOnFailure, markUnread: notifyOnFailure, isQuiet: !notifyOnFailure, reason: notifyOnFailure ? "Routine failed with error" : "Failure notification suppressed by policy", }; } // Outcome: completed if (input.notifyMode === "on_failure") { return { shouldNotify: false, markUnread: false, isQuiet: true, reason: "Successful run suppressed by on_failure policy", }; } if (input.notifyMode === "on_change") { if (!input.hasChanged) { return { shouldNotify: false, markUnread: false, isQuiet: true, reason: "No delta detected under on_change policy", }; } return { shouldNotify: true, markUnread: true, isQuiet: false, reason: "Delta detected under on_change policy", }; } // notifyMode === "always" return { shouldNotify: true, markUnread: true, isQuiet: false, reason: "Always notify policy", }; } /** * Strips verbose status chrome and formats a clean, single-sentence notification payload * compliant with AGENTS.md. */ export function formatMinimalRoutineNotification(input: { botName: string; routineName: string; outcome: "completed" | "failed"; deltaSummary?: string; error?: string; }): { title: string; body: string } { const title = input.botName.trim() || input.routineName.trim() || "Routine"; if (input.outcome === "failed") { const cleanError = input.error?.trim().slice(0, 140) || "Action needed."; return { title: `${title} failed`, body: cleanError, }; } if (input.deltaSummary?.trim()) { const cleanSummary = input.deltaSummary.trim().replace(/^[-*•\s]+/, ""); const singleSentence = cleanSummary.split(/\n/)[0]?.trim().slice(0, 140) || "Updates found."; return { title, body: singleSentence, }; } return { title, body: "New updates available.", }; }