src / anonymizer.ts
src / anonymizer.ts
import { detectAll, dedupeSpans, type Span, type SpanType } from "./detectors";
export type AnonymizeArgs = {
names?: string[];
customTerms?: string[];
configCustomTerms?: string[];
international?: boolean;
// Pre-computed spans from the optional ML model (see piiModel.ts).
// Kept out of `anonymize` itself so this primitive stays synchronous.
modelSpans?: Span[];
};
export type AnonymizeResult = {
anonymized: string;
mapping: Record<string, string>;
counts: Record<string, number>;
};
function escapeRegex(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
// If the input already contains literal pseudonyms (e.g. re-processing an
// already-redacted document), find the highest index per type so we can
// start our own counters past them and avoid output collisions.
const PSEUDO_LIKE = /\[(EMAIL|TEL|CB|NIR|IBAN|NOM|CUSTOM|ADDRESS|DATE|IDDOC)_(\d+)\]/g;
function findExistingPseudoMax(text: string): Record<string, number> {
const max: Record<string, number> = {};
for (const m of text.matchAll(PSEUDO_LIKE)) {
const type = m[1];
const n = parseInt(m[2], 10);
if (max[type] === undefined || max[type] < n) max[type] = n;
}
return max;
}
// Group different textual writings of the same logical value under one pseudonym.
// Example: "06 12 34 56 78" and "+33 6 12 34 56 78" both → "0612345678".
function normalizeValue(type: SpanType, value: string): string {
switch (type) {
case "TEL": {
// Strip the optional "(0)" first, then separators, then collapse
// the international prefix so all FR variants share one pseudonym.
let v = value.replace(/\(0\)/g, "").replace(/[ .\-]/g, "");
if (v.startsWith("+33")) v = "0" + v.slice(3);
else if (v.startsWith("0033")) v = "0" + v.slice(4);
return v;
}
case "CB":
return value.replace(/[ \-]/g, "");
case "IBAN":
return value.replace(/\s+/g, "").toUpperCase();
case "NIR":
return value.replace(/\s+/g, "").toUpperCase();
case "EMAIL":
return value.toLowerCase();
case "NOM":
return value.toLocaleLowerCase("fr");
case "ADDRESS":
// Collapse internal whitespace and case so "12 Rue X" ≡ "12 rue x".
return value.replace(/\s+/g, " ").trim().toLocaleLowerCase("fr");
case "IDDOC":
return value.replace(/\s+/g, "").toUpperCase();
case "DATE":
// Date formats are too varied to normalize safely without a parser.
// Two textual writings of the same date stay distinct pseudonyms.
return value.trim();
default:
return value;
}
}
function findLiteral(text: string, needle: string, type: SpanType, caseInsensitive: boolean): Span[] {
const trimmed = needle.trim();
if (!trimmed) return [];
const escaped = escapeRegex(trimmed);
const flags = caseInsensitive ? "giu" : "gu";
let re: RegExp;
try {
re = new RegExp(`(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])`, flags);
} catch {
re = new RegExp(escaped, caseInsensitive ? "gi" : "g");
}
const out: Span[] = [];
for (const m of text.matchAll(re)) {
out.push({ start: m.index!, end: m.index! + m[0].length, type, value: m[0] });
}
return out;
}
export function anonymize(text: string, args: AnonymizeArgs): AnonymizeResult {
// Insertion order matters: dedupeSpans is stable on length ties, so spans
// pushed first win equal-length collisions. We put validated regex spans
// first (Luhn / mod-97 checks beat probabilistic ML), then model spans,
// then caller-provided names and customs.
const spans: Span[] = [...detectAll(text, { international: args.international })];
if (args.modelSpans) spans.push(...args.modelSpans);
for (const name of args.names ?? []) {
spans.push(...findLiteral(text, name, "NOM", true));
}
const customs = [...(args.customTerms ?? []), ...(args.configCustomTerms ?? [])];
for (const term of customs) {
spans.push(...findLiteral(text, term, "CUSTOM", false));
}
const resolved = dedupeSpans(spans);
const pseudoFor = new Map<string, string>();
const firstSeen = new Map<string, string>();
// `running` counts past any pre-existing literals so generated pseudonyms
// can't collide. `added` counts only new redactions and is what the caller
// sees in the result.
const running: Record<string, number> = { ...findExistingPseudoMax(text) };
const added: Record<string, number> = {};
function keyFor(s: Span): string {
return `${s.type}|${normalizeValue(s.type, s.value)}`;
}
function getPseudo(s: Span): string {
const key = keyFor(s);
const cached = pseudoFor.get(key);
if (cached) return cached;
running[s.type] = (running[s.type] ?? 0) + 1;
added[s.type] = (added[s.type] ?? 0) + 1;
const p = `[${s.type}_${running[s.type]}]`;
pseudoFor.set(key, p);
firstSeen.set(key, s.value);
return p;
}
// Assign pseudonyms in reading order so [TYPE_1] is the first occurrence in the text.
for (const s of resolved) getPseudo(s);
// Apply replacements right-to-left to keep earlier indices valid.
const reversed = [...resolved].sort((a, b) => b.start - a.start);
let result = text;
const mapping: Record<string, string> = {};
for (const s of reversed) {
const key = keyFor(s);
const p = pseudoFor.get(key)!;
mapping[p] = firstSeen.get(key)!;
result = result.slice(0, s.start) + p + result.slice(s.end);
}
return { anonymized: result, mapping, counts: added };
}
import { detectAll, dedupeSpans, type Span, type SpanType } from "./detectors";
export type AnonymizeArgs = {
names?: string[];
customTerms?: string[];
configCustomTerms?: string[];
international?: boolean;
// Pre-computed spans from the optional ML model (see piiModel.ts).
// Kept out of `anonymize` itself so this primitive stays synchronous.
modelSpans?: Span[];
};
export type AnonymizeResult = {
anonymized: string;
mapping: Record<string, string>;
counts: Record<string, number>;
};
function escapeRegex(s: string): string {
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
// If the input already contains literal pseudonyms (e.g. re-processing an
// already-redacted document), find the highest index per type so we can
// start our own counters past them and avoid output collisions.
const PSEUDO_LIKE = /\[(EMAIL|TEL|CB|NIR|IBAN|NOM|CUSTOM|ADDRESS|DATE|IDDOC)_(\d+)\]/g;
function findExistingPseudoMax(text: string): Record<string, number> {
const max: Record<string, number> = {};
for (const m of text.matchAll(PSEUDO_LIKE)) {
const type = m[1];
const n = parseInt(m[2], 10);
if (max[type] === undefined || max[type] < n) max[type] = n;
}
return max;
}
// Group different textual writings of the same logical value under one pseudonym.
// Example: "06 12 34 56 78" and "+33 6 12 34 56 78" both → "0612345678".
function normalizeValue(type: SpanType, value: string): string {
switch (type) {
case "TEL": {
// Strip the optional "(0)" first, then separators, then collapse
// the international prefix so all FR variants share one pseudonym.
let v = value.replace(/\(0\)/g, "").replace(/[ .\-]/g, "");
if (v.startsWith("+33")) v = "0" + v.slice(3);
else if (v.startsWith("0033")) v = "0" + v.slice(4);
return v;
}
case "CB":
return value.replace(/[ \-]/g, "");
case "IBAN":
return value.replace(/\s+/g, "").toUpperCase();
case "NIR":
return value.replace(/\s+/g, "").toUpperCase();
case "EMAIL":
return value.toLowerCase();
case "NOM":
return value.toLocaleLowerCase("fr");
case "ADDRESS":
// Collapse internal whitespace and case so "12 Rue X" ≡ "12 rue x".
return value.replace(/\s+/g, " ").trim().toLocaleLowerCase("fr");
case "IDDOC":
return value.replace(/\s+/g, "").toUpperCase();
case "DATE":
// Date formats are too varied to normalize safely without a parser.
// Two textual writings of the same date stay distinct pseudonyms.
return value.trim();
default:
return value;
}
}
function findLiteral(text: string, needle: string, type: SpanType, caseInsensitive: boolean): Span[] {
const trimmed = needle.trim();
if (!trimmed) return [];
const escaped = escapeRegex(trimmed);
const flags = caseInsensitive ? "giu" : "gu";
let re: RegExp;
try {
re = new RegExp(`(?<![\\p{L}\\p{N}])${escaped}(?![\\p{L}\\p{N}])`, flags);
} catch {
re = new RegExp(escaped, caseInsensitive ? "gi" : "g");
}
const out: Span[] = [];
for (const m of text.matchAll(re)) {
out.push({ start: m.index!, end: m.index! + m[0].length, type, value: m[0] });
}
return out;
}
export function anonymize(text: string, args: AnonymizeArgs): AnonymizeResult {
// Insertion order matters: dedupeSpans is stable on length ties, so spans
// pushed first win equal-length collisions. We put validated regex spans
// first (Luhn / mod-97 checks beat probabilistic ML), then model spans,
// then caller-provided names and customs.
const spans: Span[] = [...detectAll(text, { international: args.international })];
if (args.modelSpans) spans.push(...args.modelSpans);
for (const name of args.names ?? []) {
spans.push(...findLiteral(text, name, "NOM", true));
}
const customs = [...(args.customTerms ?? []), ...(args.configCustomTerms ?? [])];
for (const term of customs) {
spans.push(...findLiteral(text, term, "CUSTOM", false));
}
const resolved = dedupeSpans(spans);
const pseudoFor = new Map<string, string>();
const firstSeen = new Map<string, string>();
// `running` counts past any pre-existing literals so generated pseudonyms
// can't collide. `added` counts only new redactions and is what the caller
// sees in the result.
const running: Record<string, number> = { ...findExistingPseudoMax(text) };
const added: Record<string, number> = {};
function keyFor(s: Span): string {
return `${s.type}|${normalizeValue(s.type, s.value)}`;
}
function getPseudo(s: Span): string {
const key = keyFor(s);
const cached = pseudoFor.get(key);
if (cached) return cached;
running[s.type] = (running[s.type] ?? 0) + 1;
added[s.type] = (added[s.type] ?? 0) + 1;
const p = `[${s.type}_${running[s.type]}]`;
pseudoFor.set(key, p);
firstSeen.set(key, s.value);
return p;
}
// Assign pseudonyms in reading order so [TYPE_1] is the first occurrence in the text.
for (const s of resolved) getPseudo(s);
// Apply replacements right-to-left to keep earlier indices valid.
const reversed = [...resolved].sort((a, b) => b.start - a.start);
let result = text;
const mapping: Record<string, string> = {};
for (const s of reversed) {
const key = keyFor(s);
const p = pseudoFor.get(key)!;
mapping[p] = firstSeen.get(key)!;
result = result.slice(0, s.start) + p + result.slice(s.end);
}
return { anonymized: result, mapping, counts: added };
}