import { type Tool, type ToolsProviderController, tool } from "@lmstudio/sdk";
import { Readability } from "@mozilla/readability";
import { Impit, type ImpitResponse } from "impit";
import { parseHTML } from "linkedom";
import { parse } from "node-html-parser";
import pThrottle from "p-throttle";
import { CookieJar } from "tough-cookie";
import TurndownService from "turndown";
import { fetchTranscript } from "youtube-transcript-plus";
import { z } from "zod";
import { configSchematics } from "./config";

type SearchResult = { title: string; url: string; snippet: string };

const cookieJar = new CookieJar();
const impit = new Impit({ browser: "chrome", ignoreTlsErrors: true, cookieJar });
const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });

td.addRule("imgToText", {
	filter: "img",
	replacement: (_, node) => {
		const alt = node.getAttribute("alt")?.trim();
		return alt ? `[Image: ${alt}]` : "";
	},
});

td.addRule("cleanLink", {
	filter: "a",
	replacement: (content, node) => {
		const text = content.replace(/\s+/g, " ").trim();
		if (!text) return "";
		const href = node.getAttribute("href");
		if (!href) return text;
		if (href.startsWith("#")) return text;
		if (href.startsWith("javascript:")) return text;
		return `[${text}](${href})`;
	},
});

async function fetchPage(url: string | URL | Request, signal: AbortSignal): Promise<ImpitResponse> {
	return impit.fetch(url, { signal, timeout: 30000 });
}

function extractContent(html: string): { title: string; content: string } {
	const { document } = parseHTML(html);
	const article = new Readability(document).parse();
	if (!article?.content) return { title: "Untitled", content: "" };
	return {
		title: article.title ?? "Untitled",
		content: td
			.turndown(article.content)
			.replace(/^[ \t]+$/gm, "")
			.replace(/\n{3,}/g, "\n\n")
			.trim(),
	};
}

export async function toolsProvider(ctl: ToolsProviderController): Promise<Tool[]> {
	// In-memory search result cache (session-scoped, LRU)
	const searchCache = new Map<string, { results: SearchResult[]; timestamp: number }>();
	const CACHE_TTL = 5 * 60 * 1000;
	const MAX_CACHE_SIZE = 100;

	const evictCacheIfNeeded = () => {
		const now = Date.now();
		for (const [key, entry] of searchCache) {
			if (now - entry.timestamp > CACHE_TTL) searchCache.delete(key);
		}
		while (searchCache.size >= MAX_CACHE_SIZE) {
			const oldest = searchCache.keys().next().value;
			if (oldest) searchCache.delete(oldest);
			else break;
		}
	};

	const getCachedResults = (query: string, pageSize: number): { results: SearchResult[] } | null => {
		const cached = searchCache.get(query);
		if (!cached) return null;
		if (Date.now() - cached.timestamp > CACHE_TTL) {
			searchCache.delete(query);
			return null;
		}
		// Move to end for LRU
		searchCache.delete(query);
		searchCache.set(query, cached);
		return { results: cached.results.slice(0, pageSize) };
	};

	const setCachedResults = (query: string, results: SearchResult[]) => {
		evictCacheIfNeeded();
		searchCache.set(query, { results, timestamp: Date.now() });
	};

	// Search: DuckDuckGo only
	const searchThrottle = pThrottle({ limit: 1, interval: 2000 });
	const searchFetch = searchThrottle(fetchPage);
	const searchDuckDuckGo = async (query: string, pageSize: number, signal: AbortSignal): Promise<SearchResult[]> => {
		const url = new URL("https://html.duckduckgo.com/html/");
		url.searchParams.append("q", query);

		const response = await searchFetch(url, signal);
		const root = parse(await response.text());
		const results: SearchResult[] = [];
		const visited = new Set<string>();
		const bodies = root.querySelectorAll(".result__body");

		for (const body of bodies) {
			if (results.length >= pageSize) break;
			const $anchor = body.querySelector("a.result__a");
			if (!$anchor) continue;
			const title = $anchor.text.replace(/\s+/g, " ").trim();
			if (!title) continue;
			let href = $anchor.getAttribute("href") || "";
			if (href.includes("aclick")) continue;
			if (href.includes("y.js")) continue;

			// Extract real URL from DDG redirect
			const uddgMatch = href.match(/[?&]uddg=([^&]+)/);
			if (uddgMatch) href = decodeURIComponent(uddgMatch[1]);

			const $snippet = body.querySelector(".result__snippet");
			const snippet = $snippet ? $snippet.text.replace(/\s+/g, " ").trim() : "";
			if (!visited.has(href)) {
				visited.add(href);
				results.push({ title, url: href, snippet });
			}
		}

		return results;
	};

	const webSearchTool = tool({
		name: "Web Search",
		description: `REQUIRED WORKFLOW: Search → Visit ONE result → Visit ONE result → Search again → Visit ONE result → Visit ONE result → Answer. You must do this exact sequence for every question. Do not skip steps or visit multiple results before searching again. After visiting 2 results from the first search, you MUST search again with a different query before visiting more results. Use 3-6 word queries. If results are poor, rephrase rather than retry the same query.`,
		parameters: { query: z.string().describe("The search query - be specific and varied across searches to get diverse results") },
		implementation: async ({ query }, { status, warn, signal }) => {
			try {
				const pageSize = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("pageSize"), 0) ?? 5;

				// Check cache first (no rate limit for cache hits)
				const cached = getCachedResults(query, pageSize);
				if (cached) {
					status(`Returning cached results for "${query}".`);
					return { results: cached.results, count: cached.results.length, cached: true };
				}

				status(`Searching DuckDuckGo for: "${query}"...`);
				let results: SearchResult[] = [];
				let lastError = "";
				const maxRetries = 3;
				for (let attempt = 1; attempt <= maxRetries; attempt++) {
					if (attempt > 1) status(`Retry ${attempt}/${maxRetries}...`);
					try {
						results = await searchDuckDuckGo(query, pageSize, signal);
						if (results.length > 0) break;
						lastError = "DuckDuckGo returned empty results";
					} catch (err) {
						const msg = err instanceof Error ? err.message : "unknown";
						lastError = `DuckDuckGo error: ${msg}`;
						break;
					}
				}

				if (results.length > 0) {
					setCachedResults(query, results);
					status(`Found ${results.length} results.`);
					return {
						results,
						count: results.length,
					};
				}

				return `No results found. ${lastError}. Try rephrasing your query.`;
			} catch (err) {
				if (err instanceof DOMException && err.name === "AbortError") return "Search was cancelled.";
				const msg = err instanceof Error ? err.message : "Unknown error";
				console.error(err);
				warn(`Search failed: ${msg}`);
				return `Error: ${msg}`;
			}
		},
	});

	const jinaThrottle = pThrottle({ limit: 1, interval: 1000 });
	const jinaFetch = jinaThrottle((url: string, signal: AbortSignal) =>
		fetch(`https://r.jina.ai/${url}`, { signal: AbortSignal.any([signal, AbortSignal.timeout(30000)]), method: "GET" }),
	);

	const visitWebsiteTool = tool({
		name: "Visit Website",
		description: `Visit a URL and extract its full text content. Read multiple sources per question — aim for 4+ visits on non-trivial topics. After reading, consider what you learned and whether you need to search for more information or verify specific claims. Seek out diverse sources and primary documentation when possible. Don't stop at the first few results — dig deeper for a thorough answer.`,
		parameters: { url: z.string().url().describe("The URL of the website to visit") },
		implementation: async ({ url }, { status, warn, signal }) => {
			const originalUrl = url;

			// De-AMP - AMP pages are always worse than the original
			url = url.replace(/\/amp\/?$/, "");
			url = url.replace(/[?&]amp=1/, "");
			const ampMatch = url.match(/google\.com\/amp\/s\/(.+)/);
			if (ampMatch) url = `https://${ampMatch[1]}`;

			// URL transformations for better content extraction
			const isMedium = /(?:www\.)?medium\.com/.test(url);
			url = url.replace(/arxiv\.org\/abs\//, "arxiv.org/pdf/");
			url = url.replace(/(?:www\.)?medium\.com/, "scribe.rawbit.ninja");
			url = url.replace(/(?:www\.)?reddit\.com/, "old.reddit.com");

			const shortUrl = url.length > 50 ? `${url.slice(0, 47)}...` : url;
			status(`Fetching content from: ${shortUrl}`);

			try {
				const contentLimit = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("contentLimit"), -1) ?? 8000;

				// Handle YouTube URLs - fetch transcript instead
				const isYouTube = url.match(/(?:youtube\.com\/watch\?.*v=|youtu\.be\/)([\w-]+)/);
				if (isYouTube) {
					status(`Fetching YouTube transcript for: ${isYouTube[1]}`);
					try {
						const { videoDetails, segments } = await fetchTranscript(url, { videoDetails: true, retries: 3, signal });
						const text = segments
							.map(t => t.text)
							.join(" ")
							.trim();
						const content = smartTruncate(text, contentLimit);
						status(`Retrieved YouTube transcript (${content.length} chars)`);
						return {
							url,
							title: videoDetails.title,
							author: videoDetails.author,
							content,
						};
					} catch (err) {
						const msg = err instanceof Error ? err.message : "unknown";
						warn(`YouTube transcript unavailable: ${msg}`);
						status("Falling back to Jina for YouTube page (content may be limited)");
					}
				}

				// PDFs always use Jina
				const isPdf = /pdf/i.test(url);
				if (isPdf) {
					const jinaResponse = await jinaFetch(url, signal);
					if (jinaResponse.ok) {
						const raw = await jinaResponse.text();
						const content = smartTruncate(raw, contentLimit);
						status(`Retrieved PDF (${content.length} chars)`);
						return { url, title: "PDF Document", content };
					}
					return `Error: Could not fetch PDF from ${url}`;
				}

				// Helper: fetch via Jina
				const tryJina = async (): Promise<{ title: string; content: string } | null> => {
					const jinaResponse = await jinaFetch(url, signal);
					if (!jinaResponse.ok) return null;
					const raw = await jinaResponse.text();
					if (raw.includes("This page maybe not yet fully loaded")) return null;
					if (raw.includes("Unavailable For Legal Reasons")) return null;
					const titleMatch = raw.match(/^Title:\s*(.+)$/m);
					const title = titleMatch ? titleMatch[1].trim() : "Untitled";
					return { title, content: smartTruncate(cleanMarkdown(raw), contentLimit) };
				};

				// Helper: fetch via Readability + Turndown
				const tryDirectFetch = async (): Promise<{ title: string; content: string } | null> => {
					try {
						const response = await fetchPage(url, signal);
						if (!response.ok) return null;
						const contentType = response.headers.get("content-type") || "";
						if (isJSON(contentType)) return { title: "JSON Response", content: await response.json() };
						const { title, content: extracted } = extractContent(await response.text());
						const content = smartTruncate(extracted, contentLimit);
						if (content.length < 2000) return null;
						return { title, content };
					} catch {
						return null;
					}
				};

				// Try direct fetch first, fallback to Jina
				let result: { title: string; content: string } | null = null;
				status("Trying direct fetch...");
				result = await tryDirectFetch();
				if (!result) {
					status("Direct fetch failed, trying Jina...");
					result = await tryJina();
				}

				// Medium fallback: if scribe.rip failed, try original URL via Jina
				if (!result && isMedium && originalUrl) {
					status("Scribe.rip failed, trying original Medium URL via Jina...");
					try {
						const fallbackResponse = await jinaFetch(originalUrl, signal);
						if (fallbackResponse.ok) {
							const raw = await fallbackResponse.text();
							const titleMatch = raw.match(/^Title:\s*(.+)$/m);
							const title = titleMatch ? titleMatch[1].trim() : "Untitled";
							const content = smartTruncate(cleanMarkdown(raw), contentLimit);
							if (content.length >= 500) {
								result = { title, content };
							}
						}
					} catch {
						/* fall through */
					}
				}

				if (!result) return `Error: Could not extract content from ${url}`;

				const { title, content } = result;
				status(`Retrieved "${title}" (${content.length} chars)`);
				return { url, title, content };
			} catch (err) {
				if (err instanceof DOMException && err.name === "AbortError") return "Website visit was cancelled.";
				const msg = err instanceof Error ? err.message : "Unknown error";
				console.error(err);
				warn(`Failed to load website: ${msg}`);
				return `Error: ${msg}`;
			}
		},
	});

	return [webSearchTool, visitWebsiteTool];
}

function isJSON(contentType: string): boolean {
	const media = contentType.split(";")[0].trim().toLowerCase();
	if (media === "application/json") return true;
	if (media.endsWith("+json")) return true;
	if (media === "text/json") return true;
	return false;
}

function undefinedIfAuto(value: unknown, autoValue: number): number | undefined {
	if (typeof value !== "number") return undefined;
	if (Number.isNaN(value)) return undefined;
	if (value === autoValue) return undefined;
	return value;
}

function cleanMarkdown(md: string): string {
	let text = md;

	// Remove Jina metadata header lines
	text = text.replace(/^(URL Source|Title|Published|Description|Markdown Content):\s*.*\n?/gm, "");

	// Remove Jina footer noise
	text = text.replace(/^(?:Let me know|Scraped|Final URL|Total|To visit).*$/gm, "");
	text = text.replace(/^-{3,}$/gm, "");

	// Convert markdown images ![alt](url) to [Image: alt] (preserve alt text as context)
	text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, (_match, alt) => (alt ? `[Image: ${alt}]` : ""));

	// Convert markdown links [text](url) to just text (preserve readable text)
	text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");

	// Remove reference-style links [text][ref]
	text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1");

	// Remove bare URLs on their own line (nav/footer links)
	text = text.replace(/^https?:\/\/\S+$/gm, "");

	// Remove HTML tags
	text = text.replace(/<[^>]+>/g, "");

	// Remove consecutive short single-word lines (nav items) but keep structural content
	text = text.replace(/^(?:\s*\w{1,20}\s*\n){4,}/gm, match => {
		const lines = match.split("\n").filter(l => l.trim());
		return lines.length > 6 ? "" : match;
	});

	// Collapse excessive blank lines
	text = text.replace(/\n{3,}/g, "\n\n");

	return text.trim();
}

function smartTruncate(text: string, limit: number): string {
	if (text.length <= limit) return text;
	const truncated = text.slice(0, limit);

	// Try paragraph boundary first
	const lastPara = truncated.lastIndexOf("\n\n");
	if (lastPara > limit * 0.7) return truncated.slice(0, lastPara).trimEnd();

	// Fall back to sentence boundary
	const lastPeriod = truncated.lastIndexOf(". ");
	const lastExclaim = truncated.lastIndexOf("! ");
	const lastQuestion = truncated.lastIndexOf("? ");
	const lastSentence = Math.max(lastPeriod, lastExclaim, lastQuestion);
	if (lastSentence > limit * 0.7) return truncated.slice(0, lastSentence + 1).trimEnd();

	return truncated;
}

import { type Tool, type ToolsProviderController, tool } from "@lmstudio/sdk";
import { Readability } from "@mozilla/readability";
import { Impit, type ImpitResponse } from "impit";
import { parseHTML } from "linkedom";
import { parse } from "node-html-parser";
import pThrottle from "p-throttle";
import { CookieJar } from "tough-cookie";
import TurndownService from "turndown";
import { fetchTranscript } from "youtube-transcript-plus";
import { z } from "zod";
import { configSchematics } from "./config";

type SearchResult = { title: string; url: string; snippet: string };

const cookieJar = new CookieJar();
const impit = new Impit({ browser: "chrome", ignoreTlsErrors: true, cookieJar });
const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });

td.addRule("imgToText", {
	filter: "img",
	replacement: (_, node) => {
		const alt = node.getAttribute("alt")?.trim();
		return alt ? `[Image: ${alt}]` : "";
	},
});

td.addRule("cleanLink", {
	filter: "a",
	replacement: (content, node) => {
		const text = content.replace(/\s+/g, " ").trim();
		if (!text) return "";
		const href = node.getAttribute("href");
		if (!href) return text;
		if (href.startsWith("#")) return text;
		if (href.startsWith("javascript:")) return text;
		return `[${text}](${href})`;
	},
});

async function fetchPage(url: string | URL | Request, signal: AbortSignal): Promise<ImpitResponse> {
	return impit.fetch(url, { signal, timeout: 30000 });
}

function extractContent(html: string): { title: string; content: string } {
	const { document } = parseHTML(html);
	const article = new Readability(document).parse();
	if (!article?.content) return { title: "Untitled", content: "" };
	return {
		title: article.title ?? "Untitled",
		content: td
			.turndown(article.content)
			.replace(/^[ \t]+$/gm, "")
			.replace(/\n{3,}/g, "\n\n")
			.trim(),
	};
}

export async function toolsProvider(ctl: ToolsProviderController): Promise<Tool[]> {
	// In-memory search result cache (session-scoped, LRU)
	const searchCache = new Map<string, { results: SearchResult[]; timestamp: number }>();
	const CACHE_TTL = 5 * 60 * 1000;
	const MAX_CACHE_SIZE = 100;

	const evictCacheIfNeeded = () => {
		const now = Date.now();
		for (const [key, entry] of searchCache) {
			if (now - entry.timestamp > CACHE_TTL) searchCache.delete(key);
		}
		while (searchCache.size >= MAX_CACHE_SIZE) {
			const oldest = searchCache.keys().next().value;
			if (oldest) searchCache.delete(oldest);
			else break;
		}
	};

	const getCachedResults = (query: string, pageSize: number): { results: SearchResult[] } | null => {
		const cached = searchCache.get(query);
		if (!cached) return null;
		if (Date.now() - cached.timestamp > CACHE_TTL) {
			searchCache.delete(query);
			return null;
		}
		// Move to end for LRU
		searchCache.delete(query);
		searchCache.set(query, cached);
		return { results: cached.results.slice(0, pageSize) };
	};

	const setCachedResults = (query: string, results: SearchResult[]) => {
		evictCacheIfNeeded();
		searchCache.set(query, { results, timestamp: Date.now() });
	};

	// Search: DuckDuckGo only
	const searchThrottle = pThrottle({ limit: 1, interval: 2000 });
	const searchFetch = searchThrottle(fetchPage);
	const searchDuckDuckGo = async (query: string, pageSize: number, signal: AbortSignal): Promise<SearchResult[]> => {
		const url = new URL("https://html.duckduckgo.com/html/");
		url.searchParams.append("q", query);

		const response = await searchFetch(url, signal);
		const root = parse(await response.text());
		const results: SearchResult[] = [];
		const visited = new Set<string>();
		const bodies = root.querySelectorAll(".result__body");

		for (const body of bodies) {
			if (results.length >= pageSize) break;
			const $anchor = body.querySelector("a.result__a");
			if (!$anchor) continue;
			const title = $anchor.text.replace(/\s+/g, " ").trim();
			if (!title) continue;
			let href = $anchor.getAttribute("href") || "";
			if (href.includes("aclick")) continue;
			if (href.includes("y.js")) continue;

			// Extract real URL from DDG redirect
			const uddgMatch = href.match(/[?&]uddg=([^&]+)/);
			if (uddgMatch) href = decodeURIComponent(uddgMatch[1]);

			const $snippet = body.querySelector(".result__snippet");
			const snippet = $snippet ? $snippet.text.replace(/\s+/g, " ").trim() : "";
			if (!visited.has(href)) {
				visited.add(href);
				results.push({ title, url: href, snippet });
			}
		}

		return results;
	};

	const webSearchTool = tool({
		name: "Web Search",
		description: `REQUIRED WORKFLOW: Search → Visit ONE result → Visit ONE result → Search again → Visit ONE result → Visit ONE result → Answer. You must do this exact sequence for every question. Do not skip steps or visit multiple results before searching again. After visiting 2 results from the first search, you MUST search again with a different query before visiting more results. Use 3-6 word queries. If results are poor, rephrase rather than retry the same query.`,
		parameters: { query: z.string().describe("The search query - be specific and varied across searches to get diverse results") },
		implementation: async ({ query }, { status, warn, signal }) => {
			try {
				const pageSize = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("pageSize"), 0) ?? 5;

				// Check cache first (no rate limit for cache hits)
				const cached = getCachedResults(query, pageSize);
				if (cached) {
					status(`Returning cached results for "${query}".`);
					return { results: cached.results, count: cached.results.length, cached: true };
				}

				status(`Searching DuckDuckGo for: "${query}"...`);
				let results: SearchResult[] = [];
				let lastError = "";
				const maxRetries = 3;
				for (let attempt = 1; attempt <= maxRetries; attempt++) {
					if (attempt > 1) status(`Retry ${attempt}/${maxRetries}...`);
					try {
						results = await searchDuckDuckGo(query, pageSize, signal);
						if (results.length > 0) break;
						lastError = "DuckDuckGo returned empty results";
					} catch (err) {
						const msg = err instanceof Error ? err.message : "unknown";
						lastError = `DuckDuckGo error: ${msg}`;
						break;
					}
				}

				if (results.length > 0) {
					setCachedResults(query, results);
					status(`Found ${results.length} results.`);
					return {
						results,
						count: results.length,
					};
				}

				return `No results found. ${lastError}. Try rephrasing your query.`;
			} catch (err) {
				if (err instanceof DOMException && err.name === "AbortError") return "Search was cancelled.";
				const msg = err instanceof Error ? err.message : "Unknown error";
				console.error(err);
				warn(`Search failed: ${msg}`);
				return `Error: ${msg}`;
			}
		},
	});

	const jinaThrottle = pThrottle({ limit: 1, interval: 1000 });
	const jinaFetch = jinaThrottle((url: string, signal: AbortSignal) =>
		fetch(`https://r.jina.ai/${url}`, { signal: AbortSignal.any([signal, AbortSignal.timeout(30000)]), method: "GET" }),
	);

	const visitWebsiteTool = tool({
		name: "Visit Website",
		description: `Visit a URL and extract its full text content. Read multiple sources per question — aim for 4+ visits on non-trivial topics. After reading, consider what you learned and whether you need to search for more information or verify specific claims. Seek out diverse sources and primary documentation when possible. Don't stop at the first few results — dig deeper for a thorough answer.`,
		parameters: { url: z.string().url().describe("The URL of the website to visit") },
		implementation: async ({ url }, { status, warn, signal }) => {
			const originalUrl = url;

			// De-AMP - AMP pages are always worse than the original
			url = url.replace(/\/amp\/?$/, "");
			url = url.replace(/[?&]amp=1/, "");
			const ampMatch = url.match(/google\.com\/amp\/s\/(.+)/);
			if (ampMatch) url = `https://${ampMatch[1]}`;

			// URL transformations for better content extraction
			const isMedium = /(?:www\.)?medium\.com/.test(url);
			url = url.replace(/arxiv\.org\/abs\//, "arxiv.org/pdf/");
			url = url.replace(/(?:www\.)?medium\.com/, "scribe.rawbit.ninja");
			url = url.replace(/(?:www\.)?reddit\.com/, "old.reddit.com");

			const shortUrl = url.length > 50 ? `${url.slice(0, 47)}...` : url;
			status(`Fetching content from: ${shortUrl}`);

			try {
				const contentLimit = undefinedIfAuto(ctl.getPluginConfig(configSchematics).get("contentLimit"), -1) ?? 8000;

				// Handle YouTube URLs - fetch transcript instead
				const isYouTube = url.match(/(?:youtube\.com\/watch\?.*v=|youtu\.be\/)([\w-]+)/);
				if (isYouTube) {
					status(`Fetching YouTube transcript for: ${isYouTube[1]}`);
					try {
						const { videoDetails, segments } = await fetchTranscript(url, { videoDetails: true, retries: 3, signal });
						const text = segments
							.map(t => t.text)
							.join(" ")
							.trim();
						const content = smartTruncate(text, contentLimit);
						status(`Retrieved YouTube transcript (${content.length} chars)`);
						return {
							url,
							title: videoDetails.title,
							author: videoDetails.author,
							content,
						};
					} catch (err) {
						const msg = err instanceof Error ? err.message : "unknown";
						warn(`YouTube transcript unavailable: ${msg}`);
						status("Falling back to Jina for YouTube page (content may be limited)");
					}
				}

				// PDFs always use Jina
				const isPdf = /pdf/i.test(url);
				if (isPdf) {
					const jinaResponse = await jinaFetch(url, signal);
					if (jinaResponse.ok) {
						const raw = await jinaResponse.text();
						const content = smartTruncate(raw, contentLimit);
						status(`Retrieved PDF (${content.length} chars)`);
						return { url, title: "PDF Document", content };
					}
					return `Error: Could not fetch PDF from ${url}`;
				}

				// Helper: fetch via Jina
				const tryJina = async (): Promise<{ title: string; content: string } | null> => {
					const jinaResponse = await jinaFetch(url, signal);
					if (!jinaResponse.ok) return null;
					const raw = await jinaResponse.text();
					if (raw.includes("This page maybe not yet fully loaded")) return null;
					if (raw.includes("Unavailable For Legal Reasons")) return null;
					const titleMatch = raw.match(/^Title:\s*(.+)$/m);
					const title = titleMatch ? titleMatch[1].trim() : "Untitled";
					return { title, content: smartTruncate(cleanMarkdown(raw), contentLimit) };
				};

				// Helper: fetch via Readability + Turndown
				const tryDirectFetch = async (): Promise<{ title: string; content: string } | null> => {
					try {
						const response = await fetchPage(url, signal);
						if (!response.ok) return null;
						const contentType = response.headers.get("content-type") || "";
						if (isJSON(contentType)) return { title: "JSON Response", content: await response.json() };
						const { title, content: extracted } = extractContent(await response.text());
						const content = smartTruncate(extracted, contentLimit);
						if (content.length < 2000) return null;
						return { title, content };
					} catch {
						return null;
					}
				};

				// Try direct fetch first, fallback to Jina
				let result: { title: string; content: string } | null = null;
				status("Trying direct fetch...");
				result = await tryDirectFetch();
				if (!result) {
					status("Direct fetch failed, trying Jina...");
					result = await tryJina();
				}

				// Medium fallback: if scribe.rip failed, try original URL via Jina
				if (!result && isMedium && originalUrl) {
					status("Scribe.rip failed, trying original Medium URL via Jina...");
					try {
						const fallbackResponse = await jinaFetch(originalUrl, signal);
						if (fallbackResponse.ok) {
							const raw = await fallbackResponse.text();
							const titleMatch = raw.match(/^Title:\s*(.+)$/m);
							const title = titleMatch ? titleMatch[1].trim() : "Untitled";
							const content = smartTruncate(cleanMarkdown(raw), contentLimit);
							if (content.length >= 500) {
								result = { title, content };
							}
						}
					} catch {
						/* fall through */
					}
				}

				if (!result) return `Error: Could not extract content from ${url}`;

				const { title, content } = result;
				status(`Retrieved "${title}" (${content.length} chars)`);
				return { url, title, content };
			} catch (err) {
				if (err instanceof DOMException && err.name === "AbortError") return "Website visit was cancelled.";
				const msg = err instanceof Error ? err.message : "Unknown error";
				console.error(err);
				warn(`Failed to load website: ${msg}`);
				return `Error: ${msg}`;
			}
		},
	});

	return [webSearchTool, visitWebsiteTool];
}

function isJSON(contentType: string): boolean {
	const media = contentType.split(";")[0].trim().toLowerCase();
	if (media === "application/json") return true;
	if (media.endsWith("+json")) return true;
	if (media === "text/json") return true;
	return false;
}

function undefinedIfAuto(value: unknown, autoValue: number): number | undefined {
	if (typeof value !== "number") return undefined;
	if (Number.isNaN(value)) return undefined;
	if (value === autoValue) return undefined;
	return value;
}

function cleanMarkdown(md: string): string {
	let text = md;

	// Remove Jina metadata header lines
	text = text.replace(/^(URL Source|Title|Published|Description|Markdown Content):\s*.*\n?/gm, "");

	// Remove Jina footer noise
	text = text.replace(/^(?:Let me know|Scraped|Final URL|Total|To visit).*$/gm, "");
	text = text.replace(/^-{3,}$/gm, "");

	// Convert markdown images ![alt](url) to [Image: alt] (preserve alt text as context)
	text = text.replace(/!\[([^\]]*)\]\([^)]+\)/g, (_match, alt) => (alt ? `[Image: ${alt}]` : ""));

	// Convert markdown links [text](url) to just text (preserve readable text)
	text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");

	// Remove reference-style links [text][ref]
	text = text.replace(/\[([^\]]+)\]\[[^\]]*\]/g, "$1");

	// Remove bare URLs on their own line (nav/footer links)
	text = text.replace(/^https?:\/\/\S+$/gm, "");

	// Remove HTML tags
	text = text.replace(/<[^>]+>/g, "");

	// Remove consecutive short single-word lines (nav items) but keep structural content
	text = text.replace(/^(?:\s*\w{1,20}\s*\n){4,}/gm, match => {
		const lines = match.split("\n").filter(l => l.trim());
		return lines.length > 6 ? "" : match;
	});

	// Collapse excessive blank lines
	text = text.replace(/\n{3,}/g, "\n\n");

	return text.trim();
}

function smartTruncate(text: string, limit: number): string {
	if (text.length <= limit) return text;
	const truncated = text.slice(0, limit);

	// Try paragraph boundary first
	const lastPara = truncated.lastIndexOf("\n\n");
	if (lastPara > limit * 0.7) return truncated.slice(0, lastPara).trimEnd();

	// Fall back to sentence boundary
	const lastPeriod = truncated.lastIndexOf(". ");
	const lastExclaim = truncated.lastIndexOf("! ");
	const lastQuestion = truncated.lastIndexOf("? ");
	const lastSentence = Math.max(lastPeriod, lastExclaim, lastQuestion);
	if (lastSentence > limit * 0.7) return truncated.slice(0, lastSentence + 1).trimEnd();

	return truncated;
}