import * as cheerio from "cheerio";
import { retryFetch } from "./retryFetch";
import { extractStructuredElements } from "./structuredExtraction";
import { normalizeUrl, resolveHref } from "../utils/url";
import { isBlockedDomain } from "../utils/domainFilter";
import { isBlockedExtension } from "../utils/blockedExtensions";
import { isNoiseUrl, pageRelevanceScore, MIN_RELEVANCE_SCORE } from "../utils/relevance";
import { MAX_LINKS, PAGE_TIMEOUT_MS, MAX_RESULT_CONTENT } from "../utils/constants";

export interface CrawlPageResult {
    url: string;
    title: string;
    content: string;
    rawHtml?: string;
    structuredContent: string[];
    links: string[];
    success: boolean;
    error?: string;
    score?: number;
    responseTime: number;
}

const crawlCache = new Map<string, CrawlPageResult>();
const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes

export function clearCrawlCache(): void {
    crawlCache.clear();
}

async function crawlPage(url: string, query: string): Promise<CrawlPageResult> {
    const startTime = Date.now();
    const normalizedUrl = normalizeUrl(url);

    const cached = crawlCache.get(normalizedUrl);
    if (cached && Date.now() - cached.responseTime < CACHE_TTL_MS) {
        return { ...cached, responseTime: Date.now() - startTime };
    }

    if (isNoiseUrl(normalizedUrl, query)) {
        const result: CrawlPageResult = {
            url: normalizedUrl, title: "Blocked", content: "", rawHtml: undefined,
            structuredContent: [], links: [], success: false,
            error: "URL identified as noise or blocked",
            responseTime: Date.now() - startTime,
        };
        crawlCache.set(normalizedUrl, result);
        return result;
    }

    let html: string;
    try {
        html = await retryFetch(normalizedUrl, { timeoutMs: PAGE_TIMEOUT_MS });
    } catch (error) {
        const message = error instanceof Error ? error.message : String(error);
        const result: CrawlPageResult = {
            url: normalizedUrl, title: "Fetch Failed", content: "", rawHtml: undefined,
            structuredContent: [], links: [], success: false,
            error: message, responseTime: Date.now() - startTime,
        };
        crawlCache.set(normalizedUrl, result);
        return result;
    }

    const $ = cheerio.load(html);
    $("script, style, noscript, iframe, svg, img, video, audio, source, link, meta").remove();
    $("body").find("[style*='display:none']").remove();
    const title = $("title").text().trim() || $("h1").first().text().trim() || "Untitled";
    const structuredContent = extractStructuredElements(html);
    const bodyText = $("body").text();
    const content = bodyText.length > MAX_RESULT_CONTENT ? bodyText.slice(0, MAX_RESULT_CONTENT) : bodyText;

    const links: string[] = [];
    const seenLinks = new Set<string>();
    $("a[href]").each((_, el) => {
        if (links.length >= MAX_LINKS) return false;
        const href = $(el).attr("href");
        if (!href) return true;
        const resolved = resolveHref(href, normalizedUrl);
        if (!resolved || seenLinks.has(resolved)) return true;
        if (isBlockedDomain(resolved) || isBlockedExtension(resolved)) return true;
        seenLinks.add(resolved);
        links.push(resolved);
    });

    const score = pageRelevanceScore(query, title, content);
    const result: CrawlPageResult = {
        url: normalizedUrl, title, content, rawHtml: html, structuredContent, links,
        success: true, score, responseTime: Date.now() - startTime,
    };

    crawlCache.set(normalizedUrl, result);
    return result;
}

export { crawlPage };

import * as cheerio from "cheerio";
import { retryFetch } from "./retryFetch";
import { extractStructuredElements } from "./structuredExtraction";
import { normalizeUrl, resolveHref } from "../utils/url";
import { isBlockedDomain } from "../utils/domainFilter";
import { isBlockedExtension } from "../utils/blockedExtensions";
import { isNoiseUrl, pageRelevanceScore, MIN_RELEVANCE_SCORE } from "../utils/relevance";
import { MAX_LINKS, PAGE_TIMEOUT_MS, MAX_RESULT_CONTENT } from "../utils/constants";

export interface CrawlPageResult {
    url: string;
    title: string;
    content: string;
    rawHtml?: string;
    structuredContent: string[];
    links: string[];
    success: boolean;
    error?: string;
    score?: number;
    responseTime: number;
}

const crawlCache = new Map<string, CrawlPageResult>();
const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes

export function clearCrawlCache(): void {
    crawlCache.clear();
}

async function crawlPage(url: string, query: string): Promise<CrawlPageResult> {
    const startTime = Date.now();
    const normalizedUrl = normalizeUrl(url);

    const cached = crawlCache.get(normalizedUrl);
    if (cached && Date.now() - cached.responseTime < CACHE_TTL_MS) {
        return { ...cached, responseTime: Date.now() - startTime };
    }

    if (isNoiseUrl(normalizedUrl, query)) {
        const result: CrawlPageResult = {
            url: normalizedUrl, title: "Blocked", content: "", rawHtml: undefined,
            structuredContent: [], links: [], success: false,
            error: "URL identified as noise or blocked",
            responseTime: Date.now() - startTime,
        };
        crawlCache.set(normalizedUrl, result);
        return result;
    }

    let html: string;
    try {
        html = await retryFetch(normalizedUrl, { timeoutMs: PAGE_TIMEOUT_MS });
    } catch (error) {
        const message = error instanceof Error ? error.message : String(error);
        const result: CrawlPageResult = {
            url: normalizedUrl, title: "Fetch Failed", content: "", rawHtml: undefined,
            structuredContent: [], links: [], success: false,
            error: message, responseTime: Date.now() - startTime,
        };
        crawlCache.set(normalizedUrl, result);
        return result;
    }

    const $ = cheerio.load(html);
    $("script, style, noscript, iframe, svg, img, video, audio, source, link, meta").remove();
    $("body").find("[style*='display:none']").remove();
    const title = $("title").text().trim() || $("h1").first().text().trim() || "Untitled";
    const structuredContent = extractStructuredElements(html);
    const bodyText = $("body").text();
    const content = bodyText.length > MAX_RESULT_CONTENT ? bodyText.slice(0, MAX_RESULT_CONTENT) : bodyText;

    const links: string[] = [];
    const seenLinks = new Set<string>();
    $("a[href]").each((_, el) => {
        if (links.length >= MAX_LINKS) return false;
        const href = $(el).attr("href");
        if (!href) return true;
        const resolved = resolveHref(href, normalizedUrl);
        if (!resolved || seenLinks.has(resolved)) return true;
        if (isBlockedDomain(resolved) || isBlockedExtension(resolved)) return true;
        seenLinks.add(resolved);
        links.push(resolved);
    });

    const score = pageRelevanceScore(query, title, content);
    const result: CrawlPageResult = {
        url: normalizedUrl, title, content, rawHtml: html, structuredContent, links,
        success: true, score, responseTime: Date.now() - startTime,
    };

    crawlCache.set(normalizedUrl, result);
    return result;
}

export { crawlPage };