src / tools / mask.ts
src / tools / mask.ts
import { tool, type Tool, type ToolsProviderController, type ToolCallContext } from "@lmstudio/sdk";
import { z } from "zod";
import { CropToolParamsShape, formatToolMetaBlock, setActiveChatContext, syncAttachmentsToState } from "../core-bundle.mjs";
import path from "path";
export function createMaskTool(ctl: ToolsProviderController): Tool {
return tool({
name: "mask",
description: `Define a region of interest on an image by drawing a CYAN bounding box.
NOTE — If you have access to detect_object: Use detectLabel to place the mask directly on a detected object's bounding box.
- detectLabel: label or array of labels to match (e.g. "cat", ["left eye", "right eye"]) — case-insensitive substring match. Comma-separated string also accepted (multi-word labels supported).
- detectIndex: zero-based index or array of indices parallel to detectLabel (default: 0 per entry). Accepts bracket notation: "[2, 4, 7]".
- frameAdjust: expand (+) or shrink (-) each bbox uniformly. Number = % of bbox diagonal; string with 'px' suffix = pixels. E.g. 5, -3, "20px".
When detectLabel is omitted and N > 1 detections exist for a label, all are auto-expanded (Option A).
When detectLabel is a single string and detectIndex lists multiple indices, masks for each index are generated (Option B).
All labels must refer to detections on the same source image.
canvas may be the original source (e.g. a1) or the annotated detect_object result (e.g. i3).
Example: canvas=i3, detectLabel=["cat", "dog"], frameAdjust=5
Per-region crop overrides (multi-region only):
cropLeft / cropRight / cropTop / cropBottom each accept a SCALAR or an ARRAY parallel to detectLabel:
- Scalar: the same override is applied to every region.
- Array: one entry per region, in the same order as detectLabel / detectIndex.
Use null for any entry to keep that region's raw detection bbox (no override for that side).
The array may be shorter than the label list — missing trailing entries behave like null.
WARNING — cropX values are IMAGE-relative (% from that image border), not mask-relative:
cropBottom=30 means "place the bottom edge of the mask 30% up from the bottom of the image".
Whether this grows or shrinks the mask depends on where the detection was:
detection cropBottom=70 (face near top) + cropBottom=30 → mask GROWS downward
detection cropBottom=10 (face near bottom) + cropBottom=30 → mask SHRINKS upward
For predictable expand/shrink relative to the detection bbox, use frameAdjust instead.
Example (pull the bottom edge of the middle mask to 10% from the image bottom):
detectLabel=["cat", "dog", "bird"], cropBottom=[null, 10, null]
→ cat and bird keep their detection bbox; dog's bottom edge is set to 10% from bottom.
Whether this grows or shrinks dog's mask depends on where dog's detection was.
For predictable expand/shrink use frameAdjust instead.
imageFormat is scalar only and applies to single-region masks; ignored in multi-region mode.
IMPORTANT — Without detect_object (iterative procedure, do NOT set all sides at once):
1. Set cropLeft only → inspect result.
2. Keep cropLeft, set cropRight only → inspect.
3. Keep both, set cropTop only → inspect.
4. Set imageFormat to lock the aspect ratio (derives cropBottom automatically).
At tight crops, 1–3% per step is enough. Symmetric values do NOT guarantee a centred subject — judge by visual gap, not numbers.
Parameters:
- canvas: Source image. Notation: a1, v2, p1, i3.
- cropLeft / cropRight / cropTop / cropBottom: Distance from that image border in % (or append px). IMAGE-relative: values are % of image dimension from that border. Valid range: 0–100 (left+right and top+bottom must each stay below 100). Negative values are invalid — they would place the edge outside the canvas. E.g. cropBottom=30 places the bottom edge 30% up from the image bottom. Scalar or per-region array (null = keep detection value). For relative expand/shrink use frameAdjust.
- imageFormat: Target aspect ratio (square=1:1, landscape=4:3, portrait=3:4, 16:9).
Only active when explicitly set. Ignored when all 4 sides are given.
Per axis: neither side → centred; one side → anchored, opposite defaults to 0; both → locked.
The result is saved as an iN image with cropLeft/cropRight/cropTop/cropBottom metadata.
When detectLabel produces multiple regions, all bounding boxes are stored (bboxes[]) and all drawn in cyan.
Pass it to inpaint or outpaint as canvas to use the marked region(s) as the mask.
Returns: The annotated image saved to the chat working directory as a new iN image.
${formatToolMetaBlock()}`,
parameters: CropToolParamsShape as Record<string, z.ZodTypeAny>,
implementation: async (args: any, ctx: ToolCallContext) => {
try {
try {
const workingDir = ctl.getWorkingDirectory();
if (typeof workingDir === "string" && workingDir.trim().length > 0) {
const chatId = path.basename(workingDir);
if (/^\d+$/.test(chatId)) {
setActiveChatContext({ chatId, workingDir, requestId: `tool-${Date.now()}` });
}
}
} catch {}
try {
const workingDir = ctl.getWorkingDirectory();
if (typeof workingDir === "string" && workingDir.trim().length > 0) {
await syncAttachmentsToState(workingDir, false, Number.MAX_SAFE_INTEGER);
}
} catch {}
const mod = await import("../core/tools.js");
const resp = await mod.handleMask(args);
if (resp && Array.isArray(resp.content)) return resp.content;
return typeof resp === "string" ? resp : JSON.stringify(resp);
} catch (e) {
return `mask failed: ${String((e as any)?.message || e)}`;
}
},
});
}
import { tool, type Tool, type ToolsProviderController, type ToolCallContext } from "@lmstudio/sdk";
import { z } from "zod";
import { CropToolParamsShape, formatToolMetaBlock, setActiveChatContext, syncAttachmentsToState } from "../core-bundle.mjs";
import path from "path";
export function createMaskTool(ctl: ToolsProviderController): Tool {
return tool({
name: "mask",
description: `Define a region of interest on an image by drawing a CYAN bounding box.
NOTE — If you have access to detect_object: Use detectLabel to place the mask directly on a detected object's bounding box.
- detectLabel: label or array of labels to match (e.g. "cat", ["left eye", "right eye"]) — case-insensitive substring match. Comma-separated string also accepted (multi-word labels supported).
- detectIndex: zero-based index or array of indices parallel to detectLabel (default: 0 per entry). Accepts bracket notation: "[2, 4, 7]".
- frameAdjust: expand (+) or shrink (-) each bbox uniformly. Number = % of bbox diagonal; string with 'px' suffix = pixels. E.g. 5, -3, "20px".
When detectLabel is omitted and N > 1 detections exist for a label, all are auto-expanded (Option A).
When detectLabel is a single string and detectIndex lists multiple indices, masks for each index are generated (Option B).
All labels must refer to detections on the same source image.
canvas may be the original source (e.g. a1) or the annotated detect_object result (e.g. i3).
Example: canvas=i3, detectLabel=["cat", "dog"], frameAdjust=5
Per-region crop overrides (multi-region only):
cropLeft / cropRight / cropTop / cropBottom each accept a SCALAR or an ARRAY parallel to detectLabel:
- Scalar: the same override is applied to every region.
- Array: one entry per region, in the same order as detectLabel / detectIndex.
Use null for any entry to keep that region's raw detection bbox (no override for that side).
The array may be shorter than the label list — missing trailing entries behave like null.
WARNING — cropX values are IMAGE-relative (% from that image border), not mask-relative:
cropBottom=30 means "place the bottom edge of the mask 30% up from the bottom of the image".
Whether this grows or shrinks the mask depends on where the detection was:
detection cropBottom=70 (face near top) + cropBottom=30 → mask GROWS downward
detection cropBottom=10 (face near bottom) + cropBottom=30 → mask SHRINKS upward
For predictable expand/shrink relative to the detection bbox, use frameAdjust instead.
Example (pull the bottom edge of the middle mask to 10% from the image bottom):
detectLabel=["cat", "dog", "bird"], cropBottom=[null, 10, null]
→ cat and bird keep their detection bbox; dog's bottom edge is set to 10% from bottom.
Whether this grows or shrinks dog's mask depends on where dog's detection was.
For predictable expand/shrink use frameAdjust instead.
imageFormat is scalar only and applies to single-region masks; ignored in multi-region mode.
IMPORTANT — Without detect_object (iterative procedure, do NOT set all sides at once):
1. Set cropLeft only → inspect result.
2. Keep cropLeft, set cropRight only → inspect.
3. Keep both, set cropTop only → inspect.
4. Set imageFormat to lock the aspect ratio (derives cropBottom automatically).
At tight crops, 1–3% per step is enough. Symmetric values do NOT guarantee a centred subject — judge by visual gap, not numbers.
Parameters:
- canvas: Source image. Notation: a1, v2, p1, i3.
- cropLeft / cropRight / cropTop / cropBottom: Distance from that image border in % (or append px). IMAGE-relative: values are % of image dimension from that border. Valid range: 0–100 (left+right and top+bottom must each stay below 100). Negative values are invalid — they would place the edge outside the canvas. E.g. cropBottom=30 places the bottom edge 30% up from the image bottom. Scalar or per-region array (null = keep detection value). For relative expand/shrink use frameAdjust.
- imageFormat: Target aspect ratio (square=1:1, landscape=4:3, portrait=3:4, 16:9).
Only active when explicitly set. Ignored when all 4 sides are given.
Per axis: neither side → centred; one side → anchored, opposite defaults to 0; both → locked.
The result is saved as an iN image with cropLeft/cropRight/cropTop/cropBottom metadata.
When detectLabel produces multiple regions, all bounding boxes are stored (bboxes[]) and all drawn in cyan.
Pass it to inpaint or outpaint as canvas to use the marked region(s) as the mask.
Returns: The annotated image saved to the chat working directory as a new iN image.
${formatToolMetaBlock()}`,
parameters: CropToolParamsShape as Record<string, z.ZodTypeAny>,
implementation: async (args: any, ctx: ToolCallContext) => {
try {
try {
const workingDir = ctl.getWorkingDirectory();
if (typeof workingDir === "string" && workingDir.trim().length > 0) {
const chatId = path.basename(workingDir);
if (/^\d+$/.test(chatId)) {
setActiveChatContext({ chatId, workingDir, requestId: `tool-${Date.now()}` });
}
}
} catch {}
try {
const workingDir = ctl.getWorkingDirectory();
if (typeof workingDir === "string" && workingDir.trim().length > 0) {
await syncAttachmentsToState(workingDir, false, Number.MAX_SAFE_INTEGER);
}
} catch {}
const mod = await import("../core/tools.js");
const resp = await mod.handleMask(args);
if (resp && Array.isArray(resp.content)) return resp.content;
return typeof resp === "string" ? resp : JSON.stringify(resp);
} catch (e) {
return `mask failed: ${String((e as any)?.message || e)}`;
}
},
});
}