import { NON_WORD_RE, normalizeWhitespace, stableHash } from "../utils/text";

const MAX_TOKEN_CACHE = 256;
const tokenCache = new Map<string, Set<string>>();

export function getTokenSet(input: string): Set<string> {
  // Hash the full input — FNV-1a is O(n) and very fast.
  // Previous code hashed only first 100 chars, causing false collisions
  // for inputs that differ after position 100.
  const key = stableHash(input);
  const cached = tokenCache.get(key);
  if (cached !== undefined) return cached;

  const tokens = new Set(
    normalizeWhitespace(input.toLowerCase())
      .split(NON_WORD_RE)
      .filter((t) => t.length > 2),
  );

  if (tokenCache.size >= MAX_TOKEN_CACHE) {
    const oldest = tokenCache.keys().next().value;
    if (oldest !== undefined) tokenCache.delete(oldest);
  }
  tokenCache.set(key, tokens);
  return tokens;
}

import { NON_WORD_RE, normalizeWhitespace, stableHash } from "../utils/text";

const MAX_TOKEN_CACHE = 256;
const tokenCache = new Map<string, Set<string>>();

export function getTokenSet(input: string): Set<string> {
  // Hash the full input — FNV-1a is O(n) and very fast.
  // Previous code hashed only first 100 chars, causing false collisions
  // for inputs that differ after position 100.
  const key = stableHash(input);
  const cached = tokenCache.get(key);
  if (cached !== undefined) return cached;

  const tokens = new Set(
    normalizeWhitespace(input.toLowerCase())
      .split(NON_WORD_RE)
      .filter((t) => t.length > 2),
  );

  if (tokenCache.size >= MAX_TOKEN_CACHE) {
    const oldest = tokenCache.keys().next().value;
    if (oldest !== undefined) tokenCache.delete(oldest);
  }
  tokenCache.set(key, tokens);
  return tokens;
}