116 lines
3.9 KiB
TypeScript
116 lines
3.9 KiB
TypeScript
import { DEFAULT_MODEL_MAX_TOKENS } from "@rakazo/contracts";
|
|
|
|
/** Hung completions must fail before the typical 5-minute run lease. */
|
|
export const MODEL_STREAM_TIMEOUT_MS = 120_000;
|
|
/**
|
|
* Local and self-hosted OpenAI-compatible servers queue requests behind a fixed
|
|
* number of GPU slots while active runs renew their lease every minute. Give
|
|
* queued turns time to wait for a free slot and prefill large contexts.
|
|
*/
|
|
export const LOCAL_MODEL_STREAM_TIMEOUT_MS = 600_000;
|
|
/** One retry keeps a transient blip from killing the turn without outliving the lease. */
|
|
export const MODEL_STREAM_MAX_RETRIES = 1;
|
|
/** Extra retries let a queued local turn survive a full multi-run slot wave. */
|
|
export const LOCAL_MODEL_STREAM_MAX_RETRIES = 3;
|
|
|
|
export function isSelfHostedModelProvider(provider: string): boolean {
|
|
return provider === "local" || provider === "openai-compatible";
|
|
}
|
|
|
|
export function resolveModelStreamTimeoutMs(provider: string, optionsTimeoutMs?: number): number {
|
|
if (typeof optionsTimeoutMs === "number" && optionsTimeoutMs > 0) return optionsTimeoutMs;
|
|
return isSelfHostedModelProvider(provider)
|
|
? LOCAL_MODEL_STREAM_TIMEOUT_MS
|
|
: MODEL_STREAM_TIMEOUT_MS;
|
|
}
|
|
|
|
export function resolveModelStreamMaxRetries(provider: string, optionsMaxRetries?: number): number {
|
|
if (typeof optionsMaxRetries === "number" && optionsMaxRetries >= 0) return optionsMaxRetries;
|
|
return isSelfHostedModelProvider(provider)
|
|
? LOCAL_MODEL_STREAM_MAX_RETRIES
|
|
: MODEL_STREAM_MAX_RETRIES;
|
|
}
|
|
|
|
export const TOOL_RESULT_TEXT_LIMIT = 12_000;
|
|
|
|
export function clipToolResultText(text: string, limit: number = TOOL_RESULT_TEXT_LIMIT): string {
|
|
return text.length > limit ? `${text.slice(0, limit)}…` : text;
|
|
}
|
|
|
|
/**
|
|
* Share one remaining character budget across all text parts.
|
|
* Later text is omitted once the aggregate limit is exhausted; non-text parts stay.
|
|
*/
|
|
export function clipToolResultContent<T>(
|
|
content: T[],
|
|
limit: number = TOOL_RESULT_TEXT_LIMIT,
|
|
): T[] {
|
|
let remaining = limit;
|
|
const clipped: T[] = [];
|
|
for (const part of content) {
|
|
if (
|
|
!part ||
|
|
typeof part !== "object" ||
|
|
!("type" in part) ||
|
|
(part as { type?: unknown }).type !== "text" ||
|
|
!("text" in part)
|
|
) {
|
|
clipped.push(part);
|
|
continue;
|
|
}
|
|
if (remaining <= 0) continue;
|
|
const text = String((part as { text: unknown }).text);
|
|
if (text.length <= remaining) {
|
|
clipped.push(part);
|
|
remaining -= text.length;
|
|
continue;
|
|
}
|
|
clipped.push({ ...part, text: clipToolResultText(text, remaining) });
|
|
remaining = 0;
|
|
}
|
|
return clipped;
|
|
}
|
|
|
|
/** Prompt tokens providers bill, including cache read/write rather than the uncached remainder. */
|
|
export function billedPromptTokens(usage: {
|
|
input?: number;
|
|
output?: number;
|
|
cacheRead?: number;
|
|
cacheWrite?: number;
|
|
}): { inputTokens: number; outputTokens: number } {
|
|
return {
|
|
inputTokens:
|
|
nonNegativeCount(usage.input) +
|
|
nonNegativeCount(usage.cacheRead) +
|
|
nonNegativeCount(usage.cacheWrite),
|
|
outputTokens: nonNegativeCount(usage.output),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Completions default to a modest output cap so OpenRouter-style providers do not
|
|
* hold credit for a model card's 128k ceiling. A configured maxTokens is the escape.
|
|
*/
|
|
export function resolveCompletionMaxTokens(
|
|
modelMaxTokens?: number,
|
|
configuredMaxTokens?: number,
|
|
optionsMaxTokens?: number,
|
|
): number {
|
|
const userCap =
|
|
typeof configuredMaxTokens === "number" && configuredMaxTokens >= 1
|
|
? configuredMaxTokens
|
|
: DEFAULT_MODEL_MAX_TOKENS;
|
|
const optionCap =
|
|
typeof optionsMaxTokens === "number" && optionsMaxTokens >= 1
|
|
? Math.min(optionsMaxTokens, userCap)
|
|
: userCap;
|
|
if (typeof modelMaxTokens === "number" && modelMaxTokens >= 1) {
|
|
return Math.min(modelMaxTokens, optionCap);
|
|
}
|
|
return optionCap;
|
|
}
|
|
|
|
function nonNegativeCount(value: number | undefined): number {
|
|
return typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : 0;
|
|
}
|