evOo-local/packages/adapters/src/web-research.ts

import type { WebSearchCategory, WebSearchHit, WebSearchRequest } from "@rakazo/adapter-kit";
import { MAX_WEB_SEARCH_QUERIES } from "./web-limits.js";

const DOMAIN_PATTERN = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,}$/i;

export const ACADEMIC_SEARCH_ENGINES = ["arxiv", "google scholar", "pubmed"] as const;
export const DISCUSSION_SEARCH_ENGINES = ["reddit", "hackernews"] as const;

const ACADEMIC_FALLBACK_DOMAINS = [
  "arxiv.org",
  "scholar.google.com",
  "pubmed.ncbi.nlm.nih.gov",
  "semanticscholar.org",
] as const;

const DISCUSSION_FALLBACK_DOMAINS = [
  "reddit.com",
  "news.ycombinator.com",
  "stackexchange.com",
] as const;

export function normalizeSearchCategory(value: unknown): WebSearchCategory | undefined {
  if (typeof value !== "string") return undefined;
  const trimmed = value.trim().toLowerCase();
  if (trimmed === "web" || trimmed === "academic" || trimmed === "discussions") {
    return trimmed;
  }
  return undefined;
}

export function normalizeSearchDomain(raw: string): string | undefined {
  const trimmed = raw.trim().toLowerCase();
  if (!trimmed) return undefined;
  let candidate = trimmed;
  if (candidate.includes("://")) {
    try {
      candidate = new URL(candidate).hostname.toLowerCase();
    } catch {
      return undefined;
    }
  } else {
    candidate = candidate.split("/")[0]?.replace(/^\*\./, "") ?? "";
  }
  candidate = candidate.replace(/^www\./, "").replace(/\.$/, "");
  return DOMAIN_PATTERN.test(candidate) ? candidate : undefined;
}

export function normalizeSearchDomains(value: unknown): string[] | undefined {
  const list = Array.isArray(value) ? value : typeof value === "string" ? [value] : [];
  const domains: string[] = [];
  const seen = new Set<string>();
  for (const item of list) {
    if (typeof item !== "string") continue;
    const normalized = normalizeSearchDomain(item);
    if (normalized && !seen.has(normalized)) {
      seen.add(normalized);
      domains.push(normalized);
    }
    if (domains.length >= 10) break;
  }
  return domains.length > 0 ? domains : undefined;
}

export function resolveSearchQueries(
  request: Pick<WebSearchRequest, "query" | "queries">,
): string[] {
  const collected: string[] = [];
  const seen = new Set<string>();
  const add = (value: unknown) => {
    if (typeof value !== "string") return;
    const trimmed = value.trim();
    if (!trimmed) return;
    const key = trimmed.toLowerCase();
    if (seen.has(key)) return;
    seen.add(key);
    collected.push(trimmed);
  };
  if (Array.isArray(request.queries)) {
    for (const q of request.queries) {
      add(q);
      if (collected.length >= MAX_WEB_SEARCH_QUERIES) break;
    }
  }
  if (collected.length < MAX_WEB_SEARCH_QUERIES) {
    add(request.query);
  }
  return collected;
}

/**
 * Applies domain and category focus operators (`site:...`) when a search backend
 * does not natively support engine/category routing (e.g. DuckDuckGo HTML).
 */
export function buildFocusedQuery(
  query: string,
  options?: { category?: WebSearchCategory; domains?: string[]; applyCategorySites?: boolean },
): string {
  const base = query.trim();
  if (!base) return "";
  const explicitDomains = options?.domains ?? [];
  const categoryDomains =
    explicitDomains.length === 0 && options?.applyCategorySites
      ? options.category === "academic"
        ? [...ACADEMIC_FALLBACK_DOMAINS]
        : options.category === "discussions"
          ? [...DISCUSSION_FALLBACK_DOMAINS]
          : []
      : [];
  const activeDomains = explicitDomains.length > 0 ? explicitDomains : categoryDomains;
  if (activeDomains.length === 0) return base;
  if (activeDomains.length === 1) {
    return `${base} site:${activeDomains[0]}`;
  }
  return `${base} (${activeDomains.map((d) => `site:${d}`).join(" OR ")})`;
}

export function normalizeUrlForDedup(rawUrl: string): string {
  try {
    const parsed = new URL(rawUrl);
    parsed.hash = "";
    const host = parsed.hostname.toLowerCase().replace(/^www\./, "");
    const path = parsed.pathname.replace(/\/+$/, "") || "/";
    return `${parsed.protocol.toLowerCase()}//${host}${path}${parsed.search}`;
  } catch {
    return rawUrl.trim();
  }
}

function tokenizeForJaccard(text: string): Set<string> {
  const set = new Set<string>();
  for (const match of text.toLowerCase().matchAll(/[\p{L}\p{N}_-]+/gu)) {
    if (match[0] && match[0].length > 2) set.add(match[0]);
  }
  return set;
}

export function jaccardSimilarity(left: string, right: string): number {
  const a = tokenizeForJaccard(left);
  const b = tokenizeForJaccard(right);
  if (a.size === 0 || b.size === 0) return 0;
  let intersection = 0;
  for (const token of a) {
    if (b.has(token)) intersection += 1;
  }
  const union = a.size + b.size - intersection;
  return union === 0 ? 0 : intersection / union;
}

/**
 * Merges and deduplicates search hits across multiple parallel queries.
 * Combines snippets when the same URL appears across queries and drops near-duplicate
 * snippets (Jaccard similarity > 0.82), modeled after Vane's researcher deduplication.
 */
export function mergeAndDeduplicateSearchHits(
  hitGroups: WebSearchHit[][],
  maxResults: number,
  allowedDomains?: string[],
): WebSearchHit[] {
  const domainSet = allowedDomains?.length ? new Set(allowedDomains) : undefined;
  const byUrl = new Map<string, WebSearchHit>();
  const ordered: WebSearchHit[] = [];

  const matchesDomainFilter = (url: string): boolean => {
    if (!domainSet) return true;
    try {
      const hostname = new URL(url).hostname.toLowerCase().replace(/^www\./, "");
      for (const domain of domainSet) {
        if (hostname === domain || hostname.endsWith(`.${domain}`)) return true;
      }
      return false;
    } catch {
      return false;
    }
  };

  const maxLen = Math.max(0, ...hitGroups.map((group) => group.length));
  // Interleave hits round-robin across query groups so every query angle is represented.
  for (let rank = 0; rank < maxLen; rank += 1) {
    for (const group of hitGroups) {
      const hit = group[rank];
      if (!hit?.url || !hit.title) continue;
      if (!matchesDomainFilter(hit.url)) continue;

      const key = normalizeUrlForDedup(hit.url);
      const existing = byUrl.get(key);
      if (existing) {
        if (
          hit.snippet &&
          !existing.snippet.includes(hit.snippet) &&
          jaccardSimilarity(existing.snippet, hit.snippet) < 0.75
        ) {
          existing.snippet = existing.snippet
            ? `${existing.snippet} … ${hit.snippet}`
            : hit.snippet;
        }
        if (!existing.author && hit.author) existing.author = hit.author;
        if (!existing.publishedDate && hit.publishedDate) {
          existing.publishedDate = hit.publishedDate;
        }
        if (!existing.engine && hit.engine) existing.engine = hit.engine;
        continue;
      }

      const nearDuplicate = ordered.some(
        (prior) =>
          prior.snippet.length > 40 &&
          hit.snippet.length > 40 &&
          jaccardSimilarity(`${prior.title} ${prior.snippet}`, `${hit.title} ${hit.snippet}`) >
            0.85,
      );
      if (nearDuplicate) continue;

      const cloned: WebSearchHit = { ...hit };
      byUrl.set(key, cloned);
      ordered.push(cloned);
    }
  }

  return ordered.slice(0, maxResults);
}

const CHUNK_SIZE_CHARS = 3_500;
const CHUNK_OVERLAP_CHARS = 350;

export function splitPageIntoChunks(
  text: string,
  chunkSize = CHUNK_SIZE_CHARS,
  overlap = CHUNK_OVERLAP_CHARS,
): string[] {
  const trimmed = text.trim();
  if (!trimmed) return [];
  if (trimmed.length <= chunkSize) return [trimmed];

  const chunks: string[] = [];
  let start = 0;
  while (start < trimmed.length) {
    let end = Math.min(trimmed.length, start + chunkSize);
    if (end < trimmed.length) {
      const window = trimmed.slice(start, end);
      const breakPos = Math.max(
        window.lastIndexOf("\n\n"),
        window.lastIndexOf("\n"),
        window.lastIndexOf(". "),
      );
      if (breakPos >= Math.floor(chunkSize * 0.5)) {
        end = start + breakPos + 1;
      }
    }
    const slice = trimmed.slice(start, end).trim();
    if (slice) chunks.push(slice);
    if (end >= trimmed.length) break;
    start = Math.max(start + 1, end - overlap);
  }
  return chunks;
}

function scoreParagraphRelevance(paragraph: string, queryTerms: string[]): number {
  const lower = paragraph.toLowerCase();
  let score = 0;
  for (const term of queryTerms) {
    if (!term) continue;
    let pos = lower.indexOf(term);
    let occurrences = 0;
    while (pos !== -1 && occurrences < 4) {
      occurrences += 1;
      pos = lower.indexOf(term, pos + term.length);
    }
    if (occurrences > 0) {
      score += 2 + Math.min(2, occurrences - 1) * 0.5;
    }
  }
  // Preserve numerical data integrity (benchmarks, percentages, tables, units) as Vane's extractor does.
  if (/\b\d+(?:\.\d+)?(?:\s*%|\s*ms|\s*s|\s*gb|\s*mb|\s*kg|\s*x\b)/i.test(paragraph)) {
    score += 0.75;
  }
  if (paragraph.includes("|") && /\|.*\|/.test(paragraph)) {
    score += 1.0;
  }
  return score;
}

/**
 * Extracts the highest-signal sections and numerical/table facts from a long page
 * relative to `query`, fitting within `maxChars` without losing late-page benchmarks or tables.
 */
export function extractQueryRelevantPageText(
  text: string,
  query: string,
  maxChars: number,
): { text: string; truncated: boolean; extracted: boolean } {
  const cleanQuery = query.trim();
  if (!cleanQuery || text.length <= maxChars) {
    const truncated = text.length > maxChars;
    return {
      text: truncated ? `${text.slice(0, maxChars)}\n\n[Content truncated]` : text,
      truncated,
      extracted: false,
    };
  }

  const queryTerms = [...tokenizeForJaccard(cleanQuery)];
  if (queryTerms.length === 0) {
    return {
      text: `${text.slice(0, maxChars)}\n\n[Content truncated]`,
      truncated: true,
      extracted: false,
    };
  }

  const blocks = text
    .split(/\n{2,}/)
    .map((block) => block.trim())
    .filter((block) => block.length > 0);

  if (blocks.length <= 1) {
    const chunks = splitPageIntoChunks(text, Math.min(2_000, Math.floor(maxChars / 2)), 200);
    const scoredChunks = chunks.map((chunk, index) => ({
      index,
      text: chunk,
      score: scoreParagraphRelevance(chunk, queryTerms) + (index === 0 ? 1.5 : 0),
    }));
    scoredChunks.sort((a, b) => b.score - a.score || a.index - b.index);
    const selected: Array<{ index: number; text: string }> = [];
    let budget = 0;
    for (const candidate of scoredChunks) {
      if (budget + candidate.text.length + 8 > maxChars) continue;
      selected.push({ index: candidate.index, text: candidate.text });
      budget += candidate.text.length + 4;
    }
    selected.sort((a, b) => a.index - b.index);
    const joined = selected.map((s) => s.text).join("\n\n…\n\n");
    return {
      text: joined || `${text.slice(0, maxChars)}\n\n[Content truncated]`,
      truncated: true,
      extracted: Boolean(joined),
    };
  }

  const scored = blocks.map((block, index) => ({
    index,
    block,
    // Keep lead summary context while prioritizing high-density query matches and tables anywhere on the page.
    score:
      scoreParagraphRelevance(block, queryTerms) + (index === 0 ? 1.25 : index === 1 ? 0.5 : 0),
  }));

  scored.sort((a, b) => b.score - a.score || a.index - b.index);

  const chosen: Array<{ index: number; block: string }> = [];
  let usedChars = 0;
  const separatorLen = 5;

  for (const item of scored) {
    if (item.score <= 0 && chosen.length > 0) break;
    const blockText =
      item.block.length > Math.floor(maxChars * 0.6)
        ? `${item.block.slice(0, Math.floor(maxChars * 0.6))}…`
        : item.block;
    if (usedChars + blockText.length + separatorLen > maxChars) {
      if (chosen.length === 0) {
        chosen.push({ index: item.index, block: blockText.slice(0, maxChars) });
      }
      continue;
    }
    chosen.push({ index: item.index, block: blockText });
    usedChars += blockText.length + separatorLen;
  }

  chosen.sort((a, b) => a.index - b.index);
  const assembledParts: string[] = [];
  let prevIndex = -1;
  for (const item of chosen) {
    if (prevIndex !== -1 && item.index > prevIndex + 1) {
      assembledParts.push("[…]");
    }
    assembledParts.push(item.block);
    prevIndex = item.index;
  }

  return {
    text: assembledParts.join("\n\n"),
    truncated: chosen.length < blocks.length,
    extracted: true,
  };
}

/**
 * Converts HTML `<table>` elements into Markdown tables in-place before Readability runs,
 * preserving numerical benchmarks and structured specs (Vane numerical data integrity rule).
 */
export function convertHtmlTablesToMarkdown(document: Document): void {
  const tables = Array.from(document.querySelectorAll("table"));
  for (const table of tables) {
    if (table.querySelector("table")) continue; // Skip outer layout wrapper tables.
    const rows = Array.from(table.querySelectorAll("tr"));
    if (rows.length === 0) continue;

    const matrix: string[][] = [];
    for (const row of rows) {
      const cells = Array.from(row.querySelectorAll("th, td")).map((cell) =>
        (cell.textContent ?? "").replace(/\s+/g, " ").replace(/\|/g, "\\|").trim(),
      );
      if (cells.some((cell) => cell.length > 0)) {
        matrix.push(cells);
      }
    }
    if (matrix.length < 2) continue;
    const colCount = Math.max(...matrix.map((r) => r.length));
    if (colCount < 2 || colCount > 16) continue;

    const padRow = (row: string[]) => {
      const padded = [...row];
      while (padded.length < colCount) padded.push("");
      return `| ${padded.join(" | ")} |`;
    };

    const header = padRow(matrix[0]!);
    const divider = `| ${Array.from({ length: colCount }, () => "---").join(" | ")} |`;
    const body = matrix.slice(1).map(padRow);
    const markdownTable = `\n\n${[header, divider, ...body].join("\n")}\n\n`;

    const replacement = document.createElement("p");
    replacement.textContent = markdownTable;
    table.replaceWith(replacement);
  }
}