// Picks the passages of a long page that matter for the current task, so the model
// reads a few thousand characters instead of the whole page. Plain lexical ranking
// (BM25 over paragraph chunks): no model call, deterministic, cheap.

const STOP = new Set(
  (
    "a an and are as at be but by can could do does did for from has have had how i if in into is it its of on or " +
    "that the their them then there these they this those to was were what when where which who whom why will with " +
    "would you your about after all also any been before being both each few more most other our out over same should " +
    "so some such than too under until up very just only own not no nor me my we us he she his her him latest current " +
    "find tell show give get know need want please"
  ).split(" ")
);

function stem(t: string): string {
  if (t.length > 4 && t.endsWith("ies")) return `${t.slice(0, -3)}y`;
  if (t.length > 3 && t.endsWith("s") && !t.endsWith("ss") && !t.endsWith("us")) return t.slice(0, -1);
  return t;
}

export function terms(text: string): string[] {
  return (text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).filter((t) => t.length > 1 && !STOP.has(t)).map(stem);
}

export interface Chunk {
  start: number;
  text: string;
  heading: string;
}

const CHUNK_MIN = 400;
const CHUNK_MAX = 1400;

function splitLong(text: string, start: number, heading: string): Chunk[] {
  const out: Chunk[] = [];
  const sentences = text.match(/[^.!?\n]+[.!?]*\s*/g) ?? [text];
  let buf = "";
  let bufStart = start;
  let pos = start;
  for (const s of sentences) {
    if (buf.length + s.length > CHUNK_MAX && buf.length >= CHUNK_MIN) {
      out.push({ start: bufStart, text: buf.trim(), heading });
      buf = "";
      bufStart = pos;
    }
    buf += s;
    pos += s.length;
  }
  if (buf.trim()) out.push({ start: bufStart, text: buf.trim(), heading });
  return out;
}

// Paragraph-based chunks of roughly 400 to 1400 characters, each remembering the
// nearest heading above it. Headings come from the extension's "#" markers.
export function chunkText(text: string): Chunk[] {
  const chunks: Chunk[] = [];
  const re = /\n{2,}/g;
  let heading = "";
  let buf = "";
  let bufStart = 0;
  let last = 0;
  const flush = () => {
    if (!buf.trim()) return;
    if (buf.length > CHUNK_MAX) chunks.push(...splitLong(buf, bufStart, heading));
    else chunks.push({ start: bufStart, text: buf.trim(), heading });
    buf = "";
  };
  const paragraphs: Array<{ start: number; text: string }> = [];
  let m: RegExpExecArray | null;
  while ((m = re.exec(text))) {
    paragraphs.push({ start: last, text: text.slice(last, m.index) });
    last = m.index + m[0].length;
  }
  paragraphs.push({ start: last, text: text.slice(last) });

  for (const p of paragraphs) {
    const t = p.text.trim();
    if (!t) continue;
    if (/^#{1,6} /.test(t)) {
      flush();
      heading = t.replace(/^#+ /, "").slice(0, 120);
      bufStart = p.start;
      buf = `${t}\n\n`;
      continue;
    }
    if (!buf) bufStart = p.start;
    buf += `${t}\n\n`;
    if (buf.length >= CHUNK_MIN) flush();
  }
  flush();
  return chunks;
}

export function outline(text: string, max = 700): string {
  const heads = text
    .split("\n")
    .filter((l) => /^#{1,4} \S/.test(l))
    .map((l) => l.trim().slice(0, 80));
  const out: string[] = [];
  let len = 0;
  for (const h of heads) {
    if (len + h.length > max) {
      out.push("...");
      break;
    }
    out.push(h);
    len += h.length + 3;
  }
  return out.join(" | ");
}

export interface FocusResult {
  passages: Chunk[];
  chars: number;
  matched: boolean;
}

// Returns the best-scoring chunks in page order, up to about `budget` characters.
// The opening chunk is included when it fits, since it usually frames the page.
export function focusPassages(text: string, goal: string, budget = 5000): FocusResult {
  const chunks = chunkText(text);
  const goalTerms = [...new Set(terms(goal))];
  if (!chunks.length || !goalTerms.length) return { passages: [], chars: 0, matched: false };

  const docs = chunks.map((c) => terms(`${c.heading} ${c.text}`));
  const avgLen = docs.reduce((n, d) => n + d.length, 0) / docs.length || 1;
  const df = new Map<string, number>();
  for (const d of docs) for (const t of new Set(d)) df.set(t, (df.get(t) ?? 0) + 1);
  const k1 = 1.2;
  const b = 0.75;
  const scores = docs.map((d, i) => {
    const tf = new Map<string, number>();
    for (const t of d) tf.set(t, (tf.get(t) ?? 0) + 1);
    let score = 0;
    for (const q of goalTerms) {
      const f = tf.get(q) ?? 0;
      if (!f) continue;
      const n = df.get(q) ?? 0;
      const idf = Math.log(1 + (chunks.length - n + 0.5) / (n + 0.5));
      score += idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * d.length) / avgLen)));
    }
    const headingTerms = new Set(terms(chunks[i].heading));
    for (const q of goalTerms) if (headingTerms.has(q)) score += 0.5;
    return score;
  });

  const ranked = scores.map((s, i) => ({ s, i })).filter((x) => x.s > 0).sort((a, b) => b.s - a.s);
  if (!ranked.length) return { passages: [], chars: 0, matched: false };

  const picked = new Set<number>();
  let used = 0;
  for (const { i } of ranked) {
    const len = chunks[i].text.length;
    if (used + len > budget && picked.size > 0) continue;
    picked.add(i);
    used += len;
    if (used >= budget) break;
  }
  if (!picked.has(0) && used + chunks[0].text.length <= budget * 1.2) {
    picked.add(0);
    used += chunks[0].text.length;
  }
  const passages = [...picked].sort((a, b) => a - b).map((i) => chunks[i]);
  return { passages, chars: used, matched: true };
}

// Picks the passages of a long page that matter for the current task, so the model
// reads a few thousand characters instead of the whole page. Plain lexical ranking
// (BM25 over paragraph chunks): no model call, deterministic, cheap.

const STOP = new Set(
  (
    "a an and are as at be but by can could do does did for from has have had how i if in into is it its of on or " +
    "that the their them then there these they this those to was were what when where which who whom why will with " +
    "would you your about after all also any been before being both each few more most other our out over same should " +
    "so some such than too under until up very just only own not no nor me my we us he she his her him latest current " +
    "find tell show give get know need want please"
  ).split(" ")
);

function stem(t: string): string {
  if (t.length > 4 && t.endsWith("ies")) return `${t.slice(0, -3)}y`;
  if (t.length > 3 && t.endsWith("s") && !t.endsWith("ss") && !t.endsWith("us")) return t.slice(0, -1);
  return t;
}

export function terms(text: string): string[] {
  return (text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).filter((t) => t.length > 1 && !STOP.has(t)).map(stem);
}

export interface Chunk {
  start: number;
  text: string;
  heading: string;
}

const CHUNK_MIN = 400;
const CHUNK_MAX = 1400;

function splitLong(text: string, start: number, heading: string): Chunk[] {
  const out: Chunk[] = [];
  const sentences = text.match(/[^.!?\n]+[.!?]*\s*/g) ?? [text];
  let buf = "";
  let bufStart = start;
  let pos = start;
  for (const s of sentences) {
    if (buf.length + s.length > CHUNK_MAX && buf.length >= CHUNK_MIN) {
      out.push({ start: bufStart, text: buf.trim(), heading });
      buf = "";
      bufStart = pos;
    }
    buf += s;
    pos += s.length;
  }
  if (buf.trim()) out.push({ start: bufStart, text: buf.trim(), heading });
  return out;
}

// Paragraph-based chunks of roughly 400 to 1400 characters, each remembering the
// nearest heading above it. Headings come from the extension's "#" markers.
export function chunkText(text: string): Chunk[] {
  const chunks: Chunk[] = [];
  const re = /\n{2,}/g;
  let heading = "";
  let buf = "";
  let bufStart = 0;
  let last = 0;
  const flush = () => {
    if (!buf.trim()) return;
    if (buf.length > CHUNK_MAX) chunks.push(...splitLong(buf, bufStart, heading));
    else chunks.push({ start: bufStart, text: buf.trim(), heading });
    buf = "";
  };
  const paragraphs: Array<{ start: number; text: string }> = [];
  let m: RegExpExecArray | null;
  while ((m = re.exec(text))) {
    paragraphs.push({ start: last, text: text.slice(last, m.index) });
    last = m.index + m[0].length;
  }
  paragraphs.push({ start: last, text: text.slice(last) });

  for (const p of paragraphs) {
    const t = p.text.trim();
    if (!t) continue;
    if (/^#{1,6} /.test(t)) {
      flush();
      heading = t.replace(/^#+ /, "").slice(0, 120);
      bufStart = p.start;
      buf = `${t}\n\n`;
      continue;
    }
    if (!buf) bufStart = p.start;
    buf += `${t}\n\n`;
    if (buf.length >= CHUNK_MIN) flush();
  }
  flush();
  return chunks;
}

export function outline(text: string, max = 700): string {
  const heads = text
    .split("\n")
    .filter((l) => /^#{1,4} \S/.test(l))
    .map((l) => l.trim().slice(0, 80));
  const out: string[] = [];
  let len = 0;
  for (const h of heads) {
    if (len + h.length > max) {
      out.push("...");
      break;
    }
    out.push(h);
    len += h.length + 3;
  }
  return out.join(" | ");
}

export interface FocusResult {
  passages: Chunk[];
  chars: number;
  matched: boolean;
}

// Returns the best-scoring chunks in page order, up to about `budget` characters.
// The opening chunk is included when it fits, since it usually frames the page.
export function focusPassages(text: string, goal: string, budget = 5000): FocusResult {
  const chunks = chunkText(text);
  const goalTerms = [...new Set(terms(goal))];
  if (!chunks.length || !goalTerms.length) return { passages: [], chars: 0, matched: false };

  const docs = chunks.map((c) => terms(`${c.heading} ${c.text}`));
  const avgLen = docs.reduce((n, d) => n + d.length, 0) / docs.length || 1;
  const df = new Map<string, number>();
  for (const d of docs) for (const t of new Set(d)) df.set(t, (df.get(t) ?? 0) + 1);
  const k1 = 1.2;
  const b = 0.75;
  const scores = docs.map((d, i) => {
    const tf = new Map<string, number>();
    for (const t of d) tf.set(t, (tf.get(t) ?? 0) + 1);
    let score = 0;
    for (const q of goalTerms) {
      const f = tf.get(q) ?? 0;
      if (!f) continue;
      const n = df.get(q) ?? 0;
      const idf = Math.log(1 + (chunks.length - n + 0.5) / (n + 0.5));
      score += idf * ((f * (k1 + 1)) / (f + k1 * (1 - b + (b * d.length) / avgLen)));
    }
    const headingTerms = new Set(terms(chunks[i].heading));
    for (const q of goalTerms) if (headingTerms.has(q)) score += 0.5;
    return score;
  });

  const ranked = scores.map((s, i) => ({ s, i })).filter((x) => x.s > 0).sort((a, b) => b.s - a.s);
  if (!ranked.length) return { passages: [], chars: 0, matched: false };

  const picked = new Set<number>();
  let used = 0;
  for (const { i } of ranked) {
    const len = chunks[i].text.length;
    if (used + len > budget && picked.size > 0) continue;
    picked.add(i);
    used += len;
    if (used >= budget) break;
  }
  if (!picked.has(0) && used + chunks[0].text.length <= budget * 1.2) {
    picked.add(0);
    used += chunks[0].text.length;
  }
  const passages = [...picked].sort((a, b) => a - b).map((i) => chunks[i]);
  return { passages, chars: used, matched: true };
}