import * as cheerio from "cheerio";

const NOISE_HEADINGS = /^(history|talk|discussion|references|sources|external links)$/i;
const NOISE_PATTERN = /(main page|discussion|view history|related changes|special pages|privacy policy|cookie|terms of service|advertisement|sponsored|talk\s+contribs|diffhist)/i;

function clean(text: string): string {
    return text.replace(/\s+/g, " ").trim();
}

function cleanWithLinks($: cheerio.CheerioAPI, $element: cheerio.Cheerio<any>): string {
    const clone = $element.clone();
    clone.find("a[href]").each((_, anchorEl) => {
        const anchor = $(anchorEl);
        const href = anchor.attr("href") || "";
        if (!href || /^(#|javascript:|mailto:|tel:|data:)/i.test(href)) return;
        const label = clean(anchor.text());
        const cleanHref = href.split('?')[0].split('#')[0];
        anchor.replaceWith(`[${label}](${cleanHref})`);
    });
    return clean(clone.text());
}

function isUsefulText(text: string, minLength = 20): boolean {
    if (!text || text.length < minLength) return false;
    if (NOISE_PATTERN.test(text)) return false;
    if (/^[-*\u2022|\\\s]+$/.test(text)) return false;
    return true;
}

function extractTableRows($: cheerio.CheerioAPI, $table: cheerio.Cheerio<any>): string[] {
    const rows: string[] = [];
    $table.find("tr").each((_, row) => {
        const cells = $(row).find("th, td").map((__, cell) => cleanWithLinks($, $(cell))).get().filter(Boolean);
        if (cells.length >= 2) rows.push(`${cells[0]}: ${cells.slice(1).join(" | ")}`);
        else if (cells.length === 1 && isUsefulText(cells[0], 10)) rows.push(cells[0]);
    });
    return rows.slice(0, 30);
}

function pushSection(sections: string[], heading: string, lines: string[]): void {
    if (lines.length === 0) return;
    const body = lines.join("\n\n").trim();
    if (body.length < 80 || NOISE_PATTERN.test(body)) return;
    sections.push(`${heading}\n\n${body}`);
}

/**
 * Single-pass extraction: walk the main content area once,
 * collecting metadata (infoboxes), sections, and tables together.
 */
export function extractStructuredElements(rawHtml: string): string[] {
    if (!rawHtml) return [];
    const $ = cheerio.load(rawHtml);
    const sections: string[] = [];
    const seen = new Set<string>();

    // Extract page metadata (infoboxes) first
    const metaSelectors = [".infobox", ".portable-infobox", "aside", "[class*='infobox']", "[class*='metadata']", "[class*='release']", "[class*='version']", "table.infobox", "table.floatright", "[style*='float:right']", "[style*='float: right']"].join(", ");
    const metadataLines: string[] = [];
    $(metaSelectors).each((_, el) => {
        const $el = $(el);
        $el.find("tr").each((__, row) => {
            const cells = $(row).children("th, td").map((___, cell) => cleanWithLinks($, $(cell))).get().filter(Boolean);
            if (cells.length < 2) return;
            const label = clean(cells[0]).replace(/^varsion type$/i, "Version Type").replace(/^release data$/i, "Release Date");
            const value = clean(cells.slice(1).join(" | "));
            if (!label || !value) return;
            const key = `${label}:${value}`;
            if (seen.has(key)) return;
            seen.add(key);
            metadataLines.push(`${label}: ${value}`);
        });
        $el.find(".pi-item, .pi-data").each((__, item) => {
            const $item = $(item);
            const label = clean(cleanWithLinks($, $item.find(".pi-data-label").first()));
            const value = clean(cleanWithLinks($, $item.find(".pi-data-value").first()));
            if (!label || !value) return;
            const key = `${label}:${value}`;
            if (seen.has(key)) return;
            seen.add(key);
            metadataLines.push(`${label}: ${value}`);
        });
    });
    if (metadataLines.length > 0) {
        sections.push(["## Page Metadata", ...metadataLines].join("\n"));
    }

    // Single pass over main content
    const $preferredMain = $("article, #mw-content-text, .mw-page-body, main, [role='main']").first();
    const $main = $preferredMain.length ? $preferredMain : $("body");
    const elements = $main.find("h2, h3, h4, p, li, table").toArray();

    let currentHeading = "## Overview";
    let currentLines: string[] = [];
    let prevTag = "";
    let seenTables = new Set<string>();

    for (const el of elements) {
        const tag = ((el as any).tagName || "").toLowerCase();
        const text = cleanWithLinks($, $(el));
        if (!text) continue;

        if (tag === "h2" || tag === "h3" || tag === "h4") {
            if (NOISE_HEADINGS.test(text) || NOISE_PATTERN.test(text)) continue;
            pushSection(sections, currentHeading, currentLines);
            currentHeading = `## ${text}`;
            currentLines = [];
            prevTag = tag;
            continue;
        }

        if (tag === "table") {
            // Deduplicate tables by fingerprint
            const tableFp = text.toLowerCase().replace(/\s+/g, "").slice(0, 100);
            if (seenTables.has(tableFp)) continue;
            seenTables.add(tableFp);

            const tableRows = extractTableRows($, $(el));
            if (tableRows.length > 0) currentLines.push(tableRows.join("\n"));
            prevTag = tag;
            continue;
        }

        if (isUsefulText(text, 25)) {
            // Avoid repeating same text from adjacent elements of the same type
            if (prevTag === tag && seenTables.has(text.toLowerCase().replace(/\s+/g, "").slice(0, 80))) {
                continue;
            }
            currentLines.push(text);
        }
        prevTag = tag;
    }

    pushSection(sections, currentHeading, currentLines);

    // Deduplicate by fingerprint
    const uniqueSections: string[] = [];
    const seenFingerprint = new Set<string>();
    for (const section of sections) {
        const fp = section.toLowerCase().replace(/\s+/g, " ").slice(0, 300);
        if (seenFingerprint.has(fp)) continue;
        seenFingerprint.add(fp);
        uniqueSections.push(section);
    }

    return uniqueSections;
}

import * as cheerio from "cheerio";

const NOISE_HEADINGS = /^(history|talk|discussion|references|sources|external links)$/i;
const NOISE_PATTERN = /(main page|discussion|view history|related changes|special pages|privacy policy|cookie|terms of service|advertisement|sponsored|talk\s+contribs|diffhist)/i;

function clean(text: string): string {
    return text.replace(/\s+/g, " ").trim();
}

function cleanWithLinks($: cheerio.CheerioAPI, $element: cheerio.Cheerio<any>): string {
    const clone = $element.clone();
    clone.find("a[href]").each((_, anchorEl) => {
        const anchor = $(anchorEl);
        const href = anchor.attr("href") || "";
        if (!href || /^(#|javascript:|mailto:|tel:|data:)/i.test(href)) return;
        const label = clean(anchor.text());
        const cleanHref = href.split('?')[0].split('#')[0];
        anchor.replaceWith(`[${label}](${cleanHref})`);
    });
    return clean(clone.text());
}

function isUsefulText(text: string, minLength = 20): boolean {
    if (!text || text.length < minLength) return false;
    if (NOISE_PATTERN.test(text)) return false;
    if (/^[-*\u2022|\\\s]+$/.test(text)) return false;
    return true;
}

function extractTableRows($: cheerio.CheerioAPI, $table: cheerio.Cheerio<any>): string[] {
    const rows: string[] = [];
    $table.find("tr").each((_, row) => {
        const cells = $(row).find("th, td").map((__, cell) => cleanWithLinks($, $(cell))).get().filter(Boolean);
        if (cells.length >= 2) rows.push(`${cells[0]}: ${cells.slice(1).join(" | ")}`);
        else if (cells.length === 1 && isUsefulText(cells[0], 10)) rows.push(cells[0]);
    });
    return rows.slice(0, 30);
}

function pushSection(sections: string[], heading: string, lines: string[]): void {
    if (lines.length === 0) return;
    const body = lines.join("\n\n").trim();
    if (body.length < 80 || NOISE_PATTERN.test(body)) return;
    sections.push(`${heading}\n\n${body}`);
}

/**
 * Single-pass extraction: walk the main content area once,
 * collecting metadata (infoboxes), sections, and tables together.
 */
export function extractStructuredElements(rawHtml: string): string[] {
    if (!rawHtml) return [];
    const $ = cheerio.load(rawHtml);
    const sections: string[] = [];
    const seen = new Set<string>();

    // Extract page metadata (infoboxes) first
    const metaSelectors = [".infobox", ".portable-infobox", "aside", "[class*='infobox']", "[class*='metadata']", "[class*='release']", "[class*='version']", "table.infobox", "table.floatright", "[style*='float:right']", "[style*='float: right']"].join(", ");
    const metadataLines: string[] = [];
    $(metaSelectors).each((_, el) => {
        const $el = $(el);
        $el.find("tr").each((__, row) => {
            const cells = $(row).children("th, td").map((___, cell) => cleanWithLinks($, $(cell))).get().filter(Boolean);
            if (cells.length < 2) return;
            const label = clean(cells[0]).replace(/^varsion type$/i, "Version Type").replace(/^release data$/i, "Release Date");
            const value = clean(cells.slice(1).join(" | "));
            if (!label || !value) return;
            const key = `${label}:${value}`;
            if (seen.has(key)) return;
            seen.add(key);
            metadataLines.push(`${label}: ${value}`);
        });
        $el.find(".pi-item, .pi-data").each((__, item) => {
            const $item = $(item);
            const label = clean(cleanWithLinks($, $item.find(".pi-data-label").first()));
            const value = clean(cleanWithLinks($, $item.find(".pi-data-value").first()));
            if (!label || !value) return;
            const key = `${label}:${value}`;
            if (seen.has(key)) return;
            seen.add(key);
            metadataLines.push(`${label}: ${value}`);
        });
    });
    if (metadataLines.length > 0) {
        sections.push(["## Page Metadata", ...metadataLines].join("\n"));
    }

    // Single pass over main content
    const $preferredMain = $("article, #mw-content-text, .mw-page-body, main, [role='main']").first();
    const $main = $preferredMain.length ? $preferredMain : $("body");
    const elements = $main.find("h2, h3, h4, p, li, table").toArray();

    let currentHeading = "## Overview";
    let currentLines: string[] = [];
    let prevTag = "";
    let seenTables = new Set<string>();

    for (const el of elements) {
        const tag = ((el as any).tagName || "").toLowerCase();
        const text = cleanWithLinks($, $(el));
        if (!text) continue;

        if (tag === "h2" || tag === "h3" || tag === "h4") {
            if (NOISE_HEADINGS.test(text) || NOISE_PATTERN.test(text)) continue;
            pushSection(sections, currentHeading, currentLines);
            currentHeading = `## ${text}`;
            currentLines = [];
            prevTag = tag;
            continue;
        }

        if (tag === "table") {
            // Deduplicate tables by fingerprint
            const tableFp = text.toLowerCase().replace(/\s+/g, "").slice(0, 100);
            if (seenTables.has(tableFp)) continue;
            seenTables.add(tableFp);

            const tableRows = extractTableRows($, $(el));
            if (tableRows.length > 0) currentLines.push(tableRows.join("\n"));
            prevTag = tag;
            continue;
        }

        if (isUsefulText(text, 25)) {
            // Avoid repeating same text from adjacent elements of the same type
            if (prevTag === tag && seenTables.has(text.toLowerCase().replace(/\s+/g, "").slice(0, 80))) {
                continue;
            }
            currentLines.push(text);
        }
        prevTag = tag;
    }

    pushSection(sections, currentHeading, currentLines);

    // Deduplicate by fingerprint
    const uniqueSections: string[] = [];
    const seenFingerprint = new Set<string>();
    for (const section of sections) {
        const fp = section.toLowerCase().replace(/\s+/g, " ").slice(0, 300);
        if (seenFingerprint.has(fp)) continue;
        seenFingerprint.add(fp);
        uniqueSections.push(section);
    }

    return uniqueSections;
}