evOo-local/packages/core/src/routine-runner.ts

/**
 * Background Routine Self-Healing, Scheduling Lifecycle & Quiet Notification Engine.
 *
 * Implements Feature F8 per PROJECT.md and ORIGINAL_REQUEST.md.
 */

import type { RoutineNotifyMode } from "./routine-policy.js";

export interface RoutineRetryConfig {
  maxAttempts: number;
  baseBackoffMs: number;
  maxBackoffMs: number;
}

export const DEFAULT_ROUTINE_RETRY_CONFIG: RoutineRetryConfig = {
  maxAttempts: 3,
  baseBackoffMs: 1_000,
  maxBackoffMs: 60_000,
};

/**
 * Classifies whether an error encountered during a routine run is transient (retryable)
 * or terminal (permanent).
 */
export function isTransientRoutineError(error: unknown): boolean {
  if (!error) return false;

  const message = error instanceof Error ? error.message : String(error);
  const code = (error as { code?: string })?.code;
  const status =
    (error as { status?: number; statusCode?: number })?.status ??
    (error as { statusCode?: number })?.statusCode;

  // 1. Explicit terminal signals (fail-fast)
  if (
    status === 401 ||
    status === 403 ||
    /unauthorized|forbidden|invalid api key|invalid token|access denied/i.test(message)
  ) {
    return false;
  }
  if (/validation failed|invalid arguments|syntax error/i.test(message)) {
    return false;
  }
  if (/budget exceeded|max steps reached|loop guard/i.test(message)) {
    return false;
  }

  // 2. HTTP transient status codes
  if (
    status === 408 ||
    status === 429 ||
    (status !== undefined && status >= 500 && status <= 504)
  ) {
    return true;
  }

  // 3. Network and socket level transient codes
  const transientNetworkCodes = new Set([
    "ECONNRESET",
    "ETIMEDOUT",
    "ECONNREFUSED",
    "EAI_AGAIN",
    "ENOTFOUND",
    "UND_ERR_CONNECT_TIMEOUT",
    "UND_ERR_SOCKET",
    "UND_ERR_HEADERS_TIMEOUT",
  ]);
  if (code && transientNetworkCodes.has(code)) {
    return true;
  }

  // 4. Undici, rate-limit, overload, and generic transient HTTP/network error strings
  if (
    /fetch failed|network timeout|socket hang up|connection reset|timed out|rate limit|too many requests|overloaded/i.test(
      message,
    ) ||
    /\b(429|500|502|503|504)\b/.test(message)
  ) {
    return true;
  }

  // 5. Database concurrency & lock contention
  if (
    code === "53300" ||
    code === "P2034" ||
    /too many database connections|deadlock detected/i.test(message)
  ) {
    return true;
  }

  // 6. Computer / Sandbox temporary busy state
  if (/computer busy|sandbox unavailable|container booting/i.test(message)) {
    return true;
  }

  return false;
}

/**
 * Calculates exponential backoff with full jitter for routine retries.
 * Prevents thundering herd using decorrelated random jitter.
 */
export function calculateRoutineBackoffMs(
  attempt: number,
  config: Partial<RoutineRetryConfig> = {},
): number {
  const cfg = { ...DEFAULT_ROUTINE_RETRY_CONFIG, ...config };
  if (attempt <= 0) return 0;

  // Exponential backoff: base * 2^(attempt - 1)
  const factor = 2 ** Math.min(attempt - 1, 6);
  const boundedMax = Math.min(cfg.maxBackoffMs, cfg.baseBackoffMs * factor);

  // Full Jitter: random value between [0, boundedMax]
  return Math.floor(Math.random() * boundedMax);
}

export interface RoutineNotificationEvaluationInput {
  routineNotifyEnabled: boolean;
  notifyMode: RoutineNotifyMode;
  hasChanged: boolean;
  outcome: "completed" | "failed";
}

export interface RoutineRunnerDecision {
  shouldNotify: boolean;
  markUnread: boolean;
  isQuiet: boolean;
  reason: string;
}

/**
 * Evaluates whether a background routine completion or failure should notify the user,
 * mark the thread unread, or execute quietly without nagging.
 */
export function evaluateRoutineNotificationPolicy(
  input: RoutineNotificationEvaluationInput,
): RoutineRunnerDecision {
  // If routine level notify flag is disabled, all notifications are muted
  if (!input.routineNotifyEnabled || input.notifyMode === "never") {
    return {
      shouldNotify: false,
      markUnread: false,
      isQuiet: true,
      reason: "Routine notifications are disabled",
    };
  }

  if (input.outcome === "failed") {
    const notifyOnFailure =
      input.notifyMode === "always" ||
      input.notifyMode === "on_failure" ||
      input.notifyMode === "on_change";
    return {
      shouldNotify: notifyOnFailure,
      markUnread: notifyOnFailure,
      isQuiet: !notifyOnFailure,
      reason: notifyOnFailure
        ? "Routine failed with error"
        : "Failure notification suppressed by policy",
    };
  }

  // Outcome: completed
  if (input.notifyMode === "on_failure") {
    return {
      shouldNotify: false,
      markUnread: false,
      isQuiet: true,
      reason: "Successful run suppressed by on_failure policy",
    };
  }

  if (input.notifyMode === "on_change") {
    if (!input.hasChanged) {
      return {
        shouldNotify: false,
        markUnread: false,
        isQuiet: true,
        reason: "No delta detected under on_change policy",
      };
    }
    return {
      shouldNotify: true,
      markUnread: true,
      isQuiet: false,
      reason: "Delta detected under on_change policy",
    };
  }

  // notifyMode === "always"
  return {
    shouldNotify: true,
    markUnread: true,
    isQuiet: false,
    reason: "Always notify policy",
  };
}

/**
 * Strips verbose status chrome and formats a clean, single-sentence notification payload
 * compliant with AGENTS.md.
 */
export function formatMinimalRoutineNotification(input: {
  botName: string;
  routineName: string;
  outcome: "completed" | "failed";
  deltaSummary?: string;
  error?: string;
}): { title: string; body: string } {
  const title = input.botName.trim() || input.routineName.trim() || "Routine";

  if (input.outcome === "failed") {
    const cleanError = input.error?.trim().slice(0, 140) || "Action needed.";
    return {
      title: `${title} failed`,
      body: cleanError,
    };
  }

  if (input.deltaSummary?.trim()) {
    const cleanSummary = input.deltaSummary.trim().replace(/^[-*•\s]+/, "");
    const singleSentence = cleanSummary.split(/\n/)[0]?.trim().slice(0, 140) || "Updates found.";
    return {
      title,
      body: singleSentence,
    };
  }

  return {
    title,
    body: "New updates available.",
  };
}