test / detector.test.js
// Detector unit tests. Compiled output is checked first; this file is run with
// plain node against dist/detector.js after `pnpm exec tsc`.
//
// These cover the Z-function detector (ported from N3). Default config:
// threshold = 10 -> zThreshold = 9, checkInterval = 32, maxTextLength = 20000.
//
// IMPORTANT SEMANTICS (matches N3 exactly):
// The Z-function compares every suffix against the buffer's START (prefix). So it
// detects: single-char runs from the start; pure-from-start periodic loops; and a
// phrase that recurs verbatim from early in the output. It does NOT detect a loop
// that begins after an unrelated preamble ("wait wait…" following "let me think…"),
// because such text shares no long common prefix with the buffer's opening. This is
// inherent to prefix-based Z detection and is preserved faithfully from N3.
const assert = require("node:assert");
const { StreamRepetitionDetector, defaultDetectorConfig } = require("../dist/detector.js");
function feed(detector, text, chunkSize = 7) {
for (let i = 0; i < text.length; i += chunkSize) {
const hit = detector.push(text.slice(i, i + chunkSize));
if (hit) return hit;
}
return null;
}
// 1. Single-char loop triggers (periodic from the start).
{
const d = new StreamRepetitionDetector();
const hit = feed(d, "x".repeat(40));
assert.ok(hit, "single-char loop should trigger");
assert.strictEqual(hit.kind, "z-function");
console.log("ok 1 single-char loop ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 2. Pure word loop from the start triggers (whitespace stripped before matching).
{
const d = new StreamRepetitionDetector();
// Periodic from position 0, so Z[3] is large and the trigger fires.
const hit = feed(d, "foo foo ".repeat(30));
assert.ok(hit, "word loop (from start) should trigger");
assert.strictEqual(hit.kind, "z-function");
console.log("ok 2 word loop ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 3. Normal prose never triggers (no >= zThreshold-char prefix recurs).
{
const d = new StreamRepetitionDetector();
const normal =
"The quick brown fox jumps over the lazy dog. In general, models that repeat " +
"themselves produce unreadable output. We should therefore guard the stream and " +
"stop early when a loop appears. This paragraph contains varied vocabulary and no " +
"consecutive identical units.";
assert.strictEqual(feed(d, normal), null, "normal prose must not trigger");
console.log("ok 3 normal prose safe");
}
// 4. Markdown table separators are stripped (ignoredChars) so decoration never triggers.
{
const d = new StreamRepetitionDetector();
// Every char in a separator row is "-" or "|" -> both are ignored, leaving nothing.
let text = "";
for (let i = 0; i < 60; i++) text += "|---|---|\n";
assert.strictEqual(feed(d, text), null, "table separators must not trigger");
console.log("ok 4 table separators safe");
}
// 5. Distinct-content rows (no real loop) stay safe even with many rows.
{
const d = new StreamRepetitionDetector();
let text = "";
for (let i = 1; i <= 20; i++) {
text += `| Row ${i} | value ${i * 7} |\n`;
}
assert.strictEqual(feed(d, text), null, "distinct rows must not trigger");
console.log("ok 5 distinct rows safe");
}
// 6. Long genuine repetition triggers (12 identical units).
{
const d = new StreamRepetitionDetector();
const hit = feed(d, "0x00\n".repeat(12));
assert.ok(hit, "repeated code-like unit should trigger");
console.log("ok 6 repeated unit ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 7. Shorter threshold still triggers a spaced loop (enough chars to pass checkInterval).
{
const d = new StreamRepetitionDetector({ threshold: 5 }); // zThreshold = 4
const hit = feed(d, "na na\n".repeat(20));
assert.ok(hit, "spaced word loop should trigger");
console.log("ok 7 spaced loop ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 8. Reset clears state (no lingering detection across turns).
{
const d = new StreamRepetitionDetector({ threshold: 5 });
feed(d, "loop ".repeat(4));
d.reset();
assert.strictEqual(feed(d, "fresh text here"), null, "after reset nothing should linger");
console.log("ok 8 reset works");
}
// 9. Default config exposes the Z-detector parameters.
{
const cfg = defaultDetectorConfig();
// ignoredChars is added by our port (N3 read this.cfg.ignoredChars without defining it).
assert.deepStrictEqual(cfg, { threshold: 10, checkInterval: 32, maxTextLength: 20000, ignoredChars: "-|" });
console.log("ok 9 default config:", JSON.stringify(cfg));
}
// 10. Post-preamble loop is NOT caught (documented N3 behavior; prefix-based Z).
{
const d = new StreamRepetitionDetector();
// "wait wait…" after an unrelated preamble shares no long prefix with the start.
assert.strictEqual(feed(d, "let me think\n" + "wait ".repeat(30)), null,
"post-preamble loop is not detected by prefix-based Z (matches N3)");
console.log("ok 10 post-preamble loop not caught (N3 semantics preserved)");
}
console.log("\nAll detector tests passed.");
test / detector.test.js
// Detector unit tests. Compiled output is checked first; this file is run with
// plain node against dist/detector.js after `pnpm exec tsc`.
//
// These cover the Z-function detector (ported from N3). Default config:
// threshold = 10 -> zThreshold = 9, checkInterval = 32, maxTextLength = 20000.
//
// IMPORTANT SEMANTICS (matches N3 exactly):
// The Z-function compares every suffix against the buffer's START (prefix). So it
// detects: single-char runs from the start; pure-from-start periodic loops; and a
// phrase that recurs verbatim from early in the output. It does NOT detect a loop
// that begins after an unrelated preamble ("wait wait…" following "let me think…"),
// because such text shares no long common prefix with the buffer's opening. This is
// inherent to prefix-based Z detection and is preserved faithfully from N3.
const assert = require("node:assert");
const { StreamRepetitionDetector, defaultDetectorConfig } = require("../dist/detector.js");
function feed(detector, text, chunkSize = 7) {
for (let i = 0; i < text.length; i += chunkSize) {
const hit = detector.push(text.slice(i, i + chunkSize));
if (hit) return hit;
}
return null;
}
// 1. Single-char loop triggers (periodic from the start).
{
const d = new StreamRepetitionDetector();
const hit = feed(d, "x".repeat(40));
assert.ok(hit, "single-char loop should trigger");
assert.strictEqual(hit.kind, "z-function");
console.log("ok 1 single-char loop ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 2. Pure word loop from the start triggers (whitespace stripped before matching).
{
const d = new StreamRepetitionDetector();
// Periodic from position 0, so Z[3] is large and the trigger fires.
const hit = feed(d, "foo foo ".repeat(30));
assert.ok(hit, "word loop (from start) should trigger");
assert.strictEqual(hit.kind, "z-function");
console.log("ok 2 word loop ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 3. Normal prose never triggers (no >= zThreshold-char prefix recurs).
{
const d = new StreamRepetitionDetector();
const normal =
"The quick brown fox jumps over the lazy dog. In general, models that repeat " +
"themselves produce unreadable output. We should therefore guard the stream and " +
"stop early when a loop appears. This paragraph contains varied vocabulary and no " +
"consecutive identical units.";
assert.strictEqual(feed(d, normal), null, "normal prose must not trigger");
console.log("ok 3 normal prose safe");
}
// 4. Markdown table separators are stripped (ignoredChars) so decoration never triggers.
{
const d = new StreamRepetitionDetector();
// Every char in a separator row is "-" or "|" -> both are ignored, leaving nothing.
let text = "";
for (let i = 0; i < 60; i++) text += "|---|---|\n";
assert.strictEqual(feed(d, text), null, "table separators must not trigger");
console.log("ok 4 table separators safe");
}
// 5. Distinct-content rows (no real loop) stay safe even with many rows.
{
const d = new StreamRepetitionDetector();
let text = "";
for (let i = 1; i <= 20; i++) {
text += `| Row ${i} | value ${i * 7} |\n`;
}
assert.strictEqual(feed(d, text), null, "distinct rows must not trigger");
console.log("ok 5 distinct rows safe");
}
// 6. Long genuine repetition triggers (12 identical units).
{
const d = new StreamRepetitionDetector();
const hit = feed(d, "0x00\n".repeat(12));
assert.ok(hit, "repeated code-like unit should trigger");
console.log("ok 6 repeated unit ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 7. Shorter threshold still triggers a spaced loop (enough chars to pass checkInterval).
{
const d = new StreamRepetitionDetector({ threshold: 5 }); // zThreshold = 4
const hit = feed(d, "na na\n".repeat(20));
assert.ok(hit, "spaced word loop should trigger");
console.log("ok 7 spaced loop ->", JSON.stringify(hit.unit), "count", hit.count);
}
// 8. Reset clears state (no lingering detection across turns).
{
const d = new StreamRepetitionDetector({ threshold: 5 });
feed(d, "loop ".repeat(4));
d.reset();
assert.strictEqual(feed(d, "fresh text here"), null, "after reset nothing should linger");
console.log("ok 8 reset works");
}
// 9. Default config exposes the Z-detector parameters.
{
const cfg = defaultDetectorConfig();
// ignoredChars is added by our port (N3 read this.cfg.ignoredChars without defining it).
assert.deepStrictEqual(cfg, { threshold: 10, checkInterval: 32, maxTextLength: 20000, ignoredChars: "-|" });
console.log("ok 9 default config:", JSON.stringify(cfg));
}
// 10. Post-preamble loop is NOT caught (documented N3 behavior; prefix-based Z).
{
const d = new StreamRepetitionDetector();
// "wait wait…" after an unrelated preamble shares no long prefix with the start.
assert.strictEqual(feed(d, "let me think\n" + "wait ".repeat(30)), null,
"post-preamble loop is not detected by prefix-based Z (matches N3)");
console.log("ok 10 post-preamble loop not caught (N3 semantics preserved)");
}
console.log("\nAll detector tests passed.");