finished item finder

.gitignore

@@ -7,6 +7,7 @@ browser-data/
 debug-screenshots/
 eng.traineddata
 .claude/
+nul
 
 # OcrDaemon build output
 tools/OcrDaemon/bin/

src/dashboard/DashboardServer.ts

@@ -4,6 +4,7 @@ import { WebSocketServer, WebSocket } from 'ws';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { logger } from '../util/logger.js';
+import { sleep } from '../util/sleep.js';
 import type { BotController } from './BotController.js';
 import type { ScreenReader } from '../game/ScreenReader.js';
 import { GRID_LAYOUTS } from '../game/GridReader.js';
@@ -203,18 +204,21 @@ export class DashboardServer {
     // Grid scan with calibrated positions
     this.app.post('/api/debug/grid-scan', async (req, res) => {
       if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
-      const { layout: layoutName } = req.body as { layout: string };
+      const { layout: layoutName, targetRow, targetCol } = req.body as { layout: string; targetRow?: number; targetCol?: number };
       if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown grid layout: ${layoutName}` }); return; }
       try {
-        const result = await this.debug.screenReader.grid.scan(layoutName);
+        const result = await this.debug.screenReader.grid.scan(layoutName, undefined, targetRow, targetCol);
         const imageBuffer = await this.debug.screenReader.captureRegion(result.layout.region);
         const imageBase64 = imageBuffer.toString('base64');
         const r = result.layout.region;
+        const matchInfo = result.matches ? `, ${result.matches.length} matches` : '';
         this.broadcastLog('info',
-          `Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height} — ${result.occupied.length} occupied cells`);
+          `Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height} — ${result.occupied.length} occupied cells${matchInfo}`);
         res.json({
           ok: true,
           occupied: result.occupied,
+          items: result.items,
+          matches: result.matches,
           cols: result.layout.cols,
           rows: result.layout.rows,
           image: imageBase64,
@@ -247,6 +251,51 @@ export class DashboardServer {
       }
     });
 
+    // Test: scan grid, find matches for target cell, hover over each for 1s
+    this.app.post('/api/debug/test-match-hover', async (req, res) => {
+      if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
+      const { layout: layoutName, targetRow, targetCol } = req.body as { layout: string; targetRow: number; targetCol: number };
+      if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown layout: ${layoutName}` }); return; }
+      if (targetRow == null || targetCol == null) { res.status(400).json({ error: 'Missing targetRow/targetCol' }); return; }
+      try {
+        // Scan with match target
+        this.broadcastLog('info', `Scanning ${layoutName} with target (${targetRow},${targetCol})...`);
+        const result = await this.debug.screenReader.grid.scan(layoutName, undefined, targetRow, targetCol);
+        const matches = result.matches ?? [];
+        const items = result.items ?? [];
+
+        // Find the item dimensions at target cell
+        const targetItem = items.find(i => targetRow >= i.row && targetRow < i.row + i.h && targetCol >= i.col && targetCol < i.col + i.w);
+        const itemSize = targetItem ? `${targetItem.w}x${targetItem.h}` : '1x1';
+        this.broadcastLog('info', `Target (${targetRow},${targetCol}) is ${itemSize}, found ${matches.length} matches`);
+
+        // Build list of cells to hover: target first, then matches
+        const hoverCells = [
+          { row: targetRow, col: targetCol, label: 'TARGET' },
+          ...matches.map(m => ({ row: m.row, col: m.col, label: `MATCH ${(m.similarity * 100).toFixed(0)}%` })),
+        ];
+
+        // Focus game and hover each cell
+        await this.debug.gameController.focusGame();
+        for (const cell of hoverCells) {
+          const center = result.layout.region;
+          const cellW = center.width / result.layout.cols;
+          const cellH = center.height / result.layout.rows;
+          const x = Math.round(center.x + cell.col * cellW + cellW / 2);
+          const y = Math.round(center.y + cell.row * cellH + cellH / 2);
+          this.broadcastLog('info', `Hovering ${cell.label} (${cell.row},${cell.col}) at (${x},${y})...`);
+          await this.debug.gameController.moveMouseTo(x, y);
+          await sleep(1000);
+        }
+
+        this.broadcastLog('info', `Done — hovered ${hoverCells.length} cells`);
+        res.json({ ok: true, itemSize, matchCount: matches.length, hoveredCount: hoverCells.length });
+      } catch (err) {
+        logger.error({ err }, 'Debug test-match-hover failed');
+        res.status(500).json({ error: 'Test match hover failed' });
+      }
+    });
+
     this.server = http.createServer(this.app);
     this.wss = new WebSocketServer({ server: this.server });
 

src/dashboard/index.html

@@ -397,6 +397,19 @@
         <button onclick="debugGridScan('shop')">Scan Shop</button>
         <button onclick="debugGridScan('vendor')">Scan Vendor</button>
       </div>
+      <div class="debug-row">
+        <select id="matchLayout" style="padding:6px 10px;background:#0d1117;border:1px solid #30363d;border-radius:6px;color:#e6edf3;font-size:13px">
+          <option value="shop">Shop</option>
+          <option value="seller">Seller</option>
+          <option value="stash12">Stash 12×12</option>
+          <option value="stash12_folder">Stash 12×12 (folder)</option>
+          <option value="inventory">Inventory</option>
+          <option value="vendor">Vendor</option>
+        </select>
+        <input type="number" id="matchRow" placeholder="Row" value="8" style="width:60px" />
+        <input type="number" id="matchCol" placeholder="Col" value="0" style="width:60px" />
+        <button class="primary" onclick="debugTestMatchHover()">Match & Hover</button>
+      </div>
       <div class="debug-row">
         <input type="text" id="debugTextInput" placeholder="Text to find (e.g. Stash, Ange)" />
         <button onclick="debugFindText()">Find</button>
@@ -645,33 +658,81 @@
   }
 
   // Debug functions
-  async function debugGridScan(layout) {
+  let lastGridLayout = null;
-    showDebugResult(`Scanning ${layout}...`);
+
+  async function debugGridScan(layout, targetRow, targetCol) {
+    lastGridLayout = layout;
+    const hasTarget = targetRow != null && targetCol != null;
+    showDebugResult(hasTarget ? `Matching (${targetRow},${targetCol}) in ${layout}...` : `Scanning ${layout}...`);
+    const body = { layout };
+    if (hasTarget) { body.targetRow = targetRow; body.targetCol = targetCol; }
     const res = await fetch('/api/debug/grid-scan', {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify({ layout }),
+      body: JSON.stringify(body),
     });
     const data = await res.json();
     if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
     const el = document.getElementById('debugResult');
     const count = data.occupied.length;
+    const items = data.items || [];
+    const matches = data.matches || [];
     const r = data.region;
     let html = `<b>${layout}</b> ${data.cols}x${data.rows}`;
-    html += ` — ${count} occupied cell(s)`;
+    html += ` — ${count} occupied, ${items.length} items`;
+    if (matches.length > 0) html += `, <span style="color:#f0883e;font-weight:bold">${matches.length} matches</span>`;
     if (r) html += `<br><span style="color:#484f58">Region: (${r.x}, ${r.y}) ${r.width}x${r.height}</span>`;
-    if (count > 0) {
+    if (items.length > 0) {
-      html += `<br>` + data.occupied.map(c => `(${c.row},${c.col})`).join(' ');
+      const sizes = {};
+      items.forEach(i => { const k = i.w + 'x' + i.h; sizes[k] = (sizes[k]||0) + 1; });
+      html += `<br><span style="color:#58a6ff">` + Object.entries(sizes).map(([k,v]) => `${v}x ${k}`).join(', ') + `</span>`;
+    }
+    if (hasTarget) {
+      html += `<br><span style="color:#f0883e">Target: (${targetRow},${targetCol})`;
+      if (matches.length > 0) html += ` → ${matches.map(m => `(${m.row},${m.col}) ${(m.similarity*100).toFixed(0)}%`).join(', ')}`;
+      html += `</span>`;
+    } else {
+      html += `<br><span style="color:#484f58">Click a cell to find matching items</span>`;
     }
     html += '<div class="grid-debug">';
     if (data.image) {
       html += `<img src="data:image/png;base64,${data.image}" alt="Grid capture" />`;
     }
+    // Build match set for highlighting
+    const matchSet = new Set(matches.map(m => m.row + ',' + m.col));
+    const targetKey = hasTarget ? targetRow + ',' + targetCol : null;
+    // Build item map: cell → item info
+    const itemMap = {};
+    const colors = ['#238636','#1f6feb','#8957e5','#da3633','#d29922','#3fb950','#388bfd','#a371f7','#f85149','#e3b341'];
+    items.forEach((item, idx) => {
+      const color = colors[idx % colors.length];
+      for (let dr = 0; dr < item.h; dr++)
+        for (let dc = 0; dc < item.w; dc++)
+          itemMap[(item.row+dr)+','+(item.col+dc)] = { item, color, isOrigin: dr===0 && dc===0 };
+    });
     html += `<div class="grid-view" style="grid-template-columns:repeat(${data.cols},12px)">`;
     const set = new Set(data.occupied.map(c => c.row + ',' + c.col));
     for (let r = 0; r < data.rows; r++) {
       for (let c = 0; c < data.cols; c++) {
-        html += `<div class="grid-cell${set.has(r+','+c) ? ' occupied' : ''}" title="(${r},${c})"></div>`;
+        const key = r+','+c;
+        const isTarget = key === targetKey;
+        const isMatch = matchSet.has(key);
+        const info = itemMap[key];
+        let bg;
+        if (isTarget) bg = '#f0883e';
+        else if (isMatch) bg = '#d29922';
+        else if (info) bg = info.color;
+        else if (set.has(key)) bg = '#238636';
+        else bg = '';
+        const outline = (isTarget || isMatch) ? 'outline:2px solid #f0883e;z-index:1;' : '';
+        const cursor = set.has(key) ? 'cursor:pointer;' : '';
+        const bgStyle = bg ? `background:${bg};` : '';
+        const style = (bgStyle || outline || cursor) ? ` style="${bgStyle}${outline}${cursor}"` : '';
+        let title = info ? `(${r},${c}) ${info.item.w}x${info.item.h}` : `(${r},${c})`;
+        if (isTarget) title += ' [TARGET]';
+        if (isMatch) { const m = matches.find(m => m.row===r && m.col===c); title += ` [MATCH ${(m.similarity*100).toFixed(0)}%]`; }
+        const onclick = set.has(key) ? ` onclick="debugGridScan('${layout}',${r},${c})"` : '';
+        html += `<div class="grid-cell${set.has(key) ? ' occupied' : ''}"${style}${onclick} title="${title}"></div>`;
       }
     }
     html += '</div></div>';
@@ -764,6 +825,22 @@
     showDebugResult(data.ok ? `Clicked at (${x}, ${y})` : `Error: ${data.error}`);
   }
 
+  async function debugTestMatchHover() {
+    const layout = document.getElementById('matchLayout').value;
+    const targetRow = parseInt(document.getElementById('matchRow').value);
+    const targetCol = parseInt(document.getElementById('matchCol').value);
+    if (isNaN(targetRow) || isNaN(targetCol)) { showDebugResult('Invalid row/col'); return; }
+    showDebugResult(`Scanning ${layout} and matching (${targetRow},${targetCol})...`);
+    const res = await fetch('/api/debug/test-match-hover', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ layout, targetRow, targetCol }),
+    });
+    const data = await res.json();
+    if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
+    showDebugResult(`Done: ${data.itemSize} item, ${data.matchCount} matches, hovered ${data.hoveredCount} cells`);
+  }
+
   function showDebugResult(text) {
     document.getElementById('debugResult').textContent = text;
   }

src/game/GameController.ts

@@ -87,6 +87,10 @@ export class GameController {
     await this.inputSender.ctrlRightClick(x, y);
   }
 
+  async moveMouseTo(x: number, y: number): Promise<void> {
+    await this.inputSender.moveMouse(x, y);
+  }
+
   async leftClickAt(x: number, y: number): Promise<void> {
     await this.inputSender.leftClick(x, y);
   }

src/game/GridReader.ts

@@ -1,5 +1,5 @@
 import { logger } from '../util/logger.js';
-import type { OcrDaemon } from './OcrDaemon.js';
+import type { OcrDaemon, GridItem, GridMatch } from './OcrDaemon.js';
 import type { Region } from '../types.js';
 
 // ── Grid type definitions ───────────────────────────────────────────────────
@@ -20,6 +20,8 @@ export interface CellCoord {
 export interface ScanResult {
   layout: GridLayout;
   occupied: CellCoord[];
+  items: GridItem[];
+  matches?: GridMatch[];
 }
 
 // ── Calibrated grid layouts (2560×1440) ─────────────────────────────────────
@@ -89,20 +91,20 @@ export class GridReader {
   /**
    * Scan a named grid layout for occupied cells.
    */
-  async scan(layoutName: string, threshold?: number): Promise<ScanResult> {
+  async scan(layoutName: string, threshold?: number, targetRow?: number, targetCol?: number): Promise<ScanResult> {
     const layout = GRID_LAYOUTS[layoutName];
     if (!layout) throw new Error(`Unknown grid layout: ${layoutName}`);
 
     const t = performance.now();
-    const occupied = await this.getOccupiedCells(layout, threshold);
+    const { occupied, items, matches } = await this.getOccupiedCells(layout, threshold, targetRow, targetCol);
 
     const ms = (performance.now() - t).toFixed(0);
     logger.info(
-      { layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, ms },
+      { layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, items: items.length, matches: matches?.length, ms },
       'Grid scan complete',
     );
 
-    return { layout, occupied };
+    return { layout, occupied, items, matches };
   }
 
   /** Get the screen-space center of a grid cell */
@@ -115,20 +117,22 @@ export class GridReader {
     };
   }
 
-  /** Scan the grid and return which cells are occupied */
+  /** Scan the grid and return which cells are occupied and detected items */
-  async getOccupiedCells(layout: GridLayout, threshold?: number): Promise<CellCoord[]> {
+  async getOccupiedCells(layout: GridLayout, threshold?: number, targetRow?: number, targetCol?: number): Promise<{ occupied: CellCoord[]; items: GridItem[]; matches?: GridMatch[] }> {
     const t = performance.now();
-    const cells = await this.daemon.gridScan(
+    const result = await this.daemon.gridScan(
       layout.region,
       layout.cols,
       layout.rows,
       threshold,
+      targetRow,
+      targetCol,
     );
 
     const occupied: CellCoord[] = [];
-    for (let row = 0; row < cells.length; row++) {
+    for (let row = 0; row < result.cells.length; row++) {
-      for (let col = 0; col < cells[row].length; col++) {
+      for (let col = 0; col < result.cells[row].length; col++) {
-        if (cells[row][col]) {
+        if (result.cells[row][col]) {
           const center = this.getCellCenter(layout, row, col);
           occupied.push({ row, col, x: center.x, y: center.y });
         }
@@ -137,10 +141,10 @@ export class GridReader {
 
     const ms = (performance.now() - t).toFixed(0);
     logger.info(
-      { layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, ms },
+      { layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, items: result.items.length, matches: result.matches?.length, ms },
       'Grid scan complete',
     );
-    return occupied;
+    return { occupied, items: result.items, matches: result.matches };
   }
 
   /** Get all cell centers in the grid */

src/game/OcrDaemon.ts

@@ -24,6 +24,25 @@ export interface OcrResponse {
   lines: OcrLine[];
 }
 
+export interface GridItem {
+  row: number;
+  col: number;
+  w: number;
+  h: number;
+}
+
+export interface GridMatch {
+  row: number;
+  col: number;
+  similarity: number;
+}
+
+export interface GridScanResult {
+  cells: boolean[][];
+  items: GridItem[];
+  matches?: GridMatch[];
+}
+
 export interface DetectGridResult {
   detected: boolean;
   region?: Region;
@@ -51,6 +70,8 @@ interface DaemonResponse {
   lines?: OcrLine[];
   image?: string;
   cells?: boolean[][];
+  items?: GridItem[];
+  matches?: GridMatch[];
   detected?: boolean;
   region?: Region;
   cols?: number;
@@ -106,11 +127,13 @@ export class OcrDaemon {
     return Buffer.from(resp.image!, 'base64');
   }
 
-  async gridScan(region: Region, cols: number, rows: number, threshold?: number): Promise<boolean[][]> {
+  async gridScan(region: Region, cols: number, rows: number, threshold?: number, targetRow?: number, targetCol?: number): Promise<GridScanResult> {
     const req: DaemonRequest = { cmd: 'grid', region, cols, rows };
     if (threshold) req.threshold = threshold;
+    if (targetRow != null && targetRow >= 0) (req as any).targetRow = targetRow;
+    if (targetCol != null && targetCol >= 0) (req as any).targetCol = targetCol;
     const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
-    return resp.cells ?? [];
+    return { cells: resp.cells ?? [], items: resp.items ?? [], matches: resp.matches ?? undefined };
   }
 
   async detectGrid(region: Region, minCellSize?: number, maxCellSize?: number): Promise<DetectGridResult> {

tools/OcrDaemon/Program.cs

@@ -129,10 +129,13 @@ void HandleCapture(Request req)
 }
 
 // Pre-loaded empty cell templates (loaded lazily on first grid scan)
+// Stored as both grayscale (for occupied detection) and ARGB (for item border detection)
 byte[]? emptyTemplate70Gray = null;
-int emptyTemplate70W = 0, emptyTemplate70H = 0;
+byte[]? emptyTemplate70Argb = null;
+int emptyTemplate70W = 0, emptyTemplate70H = 0, emptyTemplate70Stride = 0;
 byte[]? emptyTemplate35Gray = null;
-int emptyTemplate35W = 0, emptyTemplate35H = 0;
+byte[]? emptyTemplate35Argb = null;
+int emptyTemplate35W = 0, emptyTemplate35H = 0, emptyTemplate35Stride = 0;
 
 void LoadTemplatesIfNeeded()
 {
@@ -150,23 +153,23 @@ void LoadTemplatesIfNeeded()
         using var bmp = new Bitmap(t70Path);
         emptyTemplate70W = bmp.Width;
         emptyTemplate70H = bmp.Height;
-        emptyTemplate70Gray = BitmapToGray(bmp);
+        (emptyTemplate70Gray, emptyTemplate70Argb, emptyTemplate70Stride) = BitmapToGrayAndArgb(bmp);
     }
     if (System.IO.File.Exists(t35Path))
     {
         using var bmp = new Bitmap(t35Path);
         emptyTemplate35W = bmp.Width;
         emptyTemplate35H = bmp.Height;
-        emptyTemplate35Gray = BitmapToGray(bmp);
+        (emptyTemplate35Gray, emptyTemplate35Argb, emptyTemplate35Stride) = BitmapToGrayAndArgb(bmp);
     }
 }
 
-byte[] BitmapToGray(Bitmap bmp)
+(byte[] gray, byte[] argb, int stride) BitmapToGrayAndArgb(Bitmap bmp)
 {
     int w = bmp.Width, h = bmp.Height;
     var data = bmp.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
-    byte[] pixels = new byte[data.Stride * h];
+    byte[] argb = new byte[data.Stride * h];
-    Marshal.Copy(data.Scan0, pixels, 0, pixels.Length);
+    Marshal.Copy(data.Scan0, argb, 0, argb.Length);
     bmp.UnlockBits(data);
     int stride = data.Stride;
 
@@ -175,8 +178,14 @@ byte[] BitmapToGray(Bitmap bmp)
         for (int x = 0; x < w; x++)
         {
             int i = y * stride + x * 4;
-            gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
+            gray[y * w + x] = (byte)((argb[i] + argb[i + 1] + argb[i + 2]) / 3);
         }
+    return (gray, argb, stride);
+}
+
+byte[] BitmapToGray(Bitmap bmp)
+{
+    var (gray, _, _) = BitmapToGrayAndArgb(bmp);
     return gray;
 }
 
@@ -199,18 +208,23 @@ void HandleGrid(Request req)
     // Pick the right empty template based on cell size
     int nominalCell = (int)Math.Round(cellW);
     byte[]? templateGray;
-    int templateW, templateH;
+    byte[]? templateArgb;
+    int templateW, templateH, templateStride;
     if (nominalCell <= 40 && emptyTemplate35Gray != null)
     {
         templateGray = emptyTemplate35Gray;
+        templateArgb = emptyTemplate35Argb!;
         templateW = emptyTemplate35W;
         templateH = emptyTemplate35H;
+        templateStride = emptyTemplate35Stride;
     }
     else if (emptyTemplate70Gray != null)
     {
         templateGray = emptyTemplate70Gray;
+        templateArgb = emptyTemplate70Argb!;
         templateW = emptyTemplate70W;
         templateH = emptyTemplate70H;
+        templateStride = emptyTemplate70Stride;
     }
     else
     {
@@ -218,8 +232,8 @@ void HandleGrid(Request req)
         return;
     }
 
-    // Convert captured bitmap to grayscale
+    // Convert captured bitmap to grayscale + keep ARGB for border color comparison
-    byte[] captureGray = BitmapToGray(bitmap);
+    var (captureGray, captureArgb, captureStride) = BitmapToGrayAndArgb(bitmap);
     int captureW = bitmap.Width;
 
     // Border to skip (outer pixels may differ between cells)
@@ -277,7 +291,209 @@ void HandleGrid(Request req)
         if (debug) Console.Error.WriteLine($"  Row {row,2}: {string.Join(" ", debugDiffs)}");
     }
 
-    WriteResponse(new GridResponse { Cells = cells });
+    // ── Item detection: compare border pixels to empty template (grayscale) ──
+    // Items have a colored tint behind them that shows through grid lines.
+    // Compare each cell's border strip against the template's border pixels.
+    // If they differ → item tint present → cells belong to same item.
+    int[] parent = new int[rows * cols];
+    for (int i = 0; i < parent.Length; i++) parent[i] = i;
+
+    int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; }
+    void Union(int a, int b) { parent[Find(a)] = Find(b); }
+
+    int stripWidth = Math.Max(2, border / 2);
+    int stripInset = (int)(cellW * 0.15);
+    double borderDiffThresh = 15.0;
+
+    for (int row = 0; row < rows; row++)
+    {
+        for (int col = 0; col < cols; col++)
+        {
+            if (!cells[row][col]) continue;
+            int cx0 = (int)(col * cellW);
+            int cy0 = (int)(row * cellH);
+
+            // Check right neighbor
+            if (col + 1 < cols && cells[row][col + 1])
+            {
+                long diffSum = 0; int cnt = 0;
+                int xStart = (int)((col + 1) * cellW) - stripWidth;
+                int yFrom = cy0 + stripInset;
+                int yTo = (int)((row + 1) * cellH) - stripInset;
+                for (int sy = yFrom; sy < yTo; sy += 2)
+                {
+                    int tmplY = sy - cy0;
+                    for (int sx = xStart; sx < xStart + stripWidth * 2; sx++)
+                    {
+                        if (sx < 0 || sx >= captureW) continue;
+                        int tmplX = sx - cx0;
+                        if (tmplX < 0 || tmplX >= templateW) continue;
+                        diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
+                        cnt++;
+                    }
+                }
+                double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
+                if (debug) Console.Error.WriteLine($"    H ({row},{col})->({row},{col+1}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
+                if (meanDiff > borderDiffThresh)
+                    Union(row * cols + col, row * cols + col + 1);
+            }
+
+            // Check bottom neighbor
+            if (row + 1 < rows && cells[row + 1][col])
+            {
+                long diffSum = 0; int cnt = 0;
+                int yStart = (int)((row + 1) * cellH) - stripWidth;
+                int xFrom = cx0 + stripInset;
+                int xTo = (int)((col + 1) * cellW) - stripInset;
+                for (int sx = xFrom; sx < xTo; sx += 2)
+                {
+                    int tmplX = sx - cx0;
+                    for (int sy = yStart; sy < yStart + stripWidth * 2; sy++)
+                    {
+                        if (sy < 0 || sy >= bitmap.Height) continue;
+                        int tmplY = sy - cy0;
+                        if (tmplY < 0 || tmplY >= templateH) continue;
+                        diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
+                        cnt++;
+                    }
+                }
+                double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
+                if (debug) Console.Error.WriteLine($"    V ({row},{col})->({row+1},{col}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
+                if (meanDiff > borderDiffThresh)
+                    Union(row * cols + col, (row + 1) * cols + col);
+            }
+        }
+    }
+
+    // Extract items from union-find groups
+    var groups = new Dictionary<int, List<(int row, int col)>>();
+    for (int row = 0; row < rows; row++)
+        for (int col = 0; col < cols; col++)
+            if (cells[row][col])
+            {
+                int root = Find(row * cols + col);
+                if (!groups.ContainsKey(root)) groups[root] = [];
+                groups[root].Add((row, col));
+            }
+
+    var items = new List<GridItem>();
+    foreach (var group in groups.Values)
+    {
+        int minR = group.Min(c => c.row);
+        int maxR = group.Max(c => c.row);
+        int minC = group.Min(c => c.col);
+        int maxC = group.Max(c => c.col);
+        items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 });
+    }
+
+    if (debug)
+    {
+        Console.Error.WriteLine($"  Items found: {items.Count}");
+        foreach (var item in items)
+            Console.Error.WriteLine($"    ({item.Row},{item.Col}) {item.W}x{item.H}");
+    }
+
+    // ── Visual matching: find cells similar to target ──
+    List<GridMatch>? matches = null;
+    if (req.TargetRow >= 0 && req.TargetCol >= 0 &&
+        req.TargetRow < rows && req.TargetCol < cols &&
+        cells[req.TargetRow][req.TargetCol])
+    {
+        matches = FindMatchingCells(
+            captureGray, captureW, bitmap.Height,
+            cells, rows, cols, cellW, cellH, border,
+            req.TargetRow, req.TargetCol, debug);
+    }
+
+    WriteResponse(new GridResponse { Cells = cells, Items = items, Matches = matches });
+}
+
+/// Find all occupied cells visually similar to the target cell using full-resolution NCC.
+/// Full resolution gives better discrimination — sockets are a small fraction of total pixels.
+List<GridMatch> FindMatchingCells(
+    byte[] gray, int imgW, int imgH,
+    List<List<bool>> cells, int rows, int cols,
+    float cellW, float cellH, int border,
+    int targetRow, int targetCol, bool debug)
+{
+    int innerW = (int)cellW - border * 2;
+    int innerH = (int)cellH - border * 2;
+    if (innerW <= 4 || innerH <= 4) return [];
+
+    int tCx0 = (int)(targetCol * cellW) + border;
+    int tCy0 = (int)(targetRow * cellH) + border;
+    int tInnerW = Math.Min(innerW, imgW - tCx0);
+    int tInnerH = Math.Min(innerH, imgH - tCy0);
+    if (tInnerW < innerW || tInnerH < innerH) return [];
+
+    int n = innerW * innerH;
+
+    // Pre-compute target cell pixels and stats
+    double[] targetPixels = new double[n];
+    double tMean = 0;
+    for (int py = 0; py < innerH; py++)
+        for (int px = 0; px < innerW; px++)
+        {
+            double v = gray[(tCy0 + py) * imgW + (tCx0 + px)];
+            targetPixels[py * innerW + px] = v;
+            tMean += v;
+        }
+    tMean /= n;
+
+    double tStd = 0;
+    for (int i = 0; i < n; i++)
+        tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean);
+    tStd = Math.Sqrt(tStd / n);
+
+    if (debug) Console.Error.WriteLine($"  Match target ({targetRow},{targetCol}): {innerW}x{innerH} ({n}px), mean={tMean:F1}, std={tStd:F1}");
+    if (tStd < 3.0) return [];
+
+    double matchThreshold = 0.70;
+    var matches = new List<GridMatch>();
+
+    for (int row = 0; row < rows; row++)
+    {
+        for (int col = 0; col < cols; col++)
+        {
+            if (!cells[row][col]) continue;
+            if (row == targetRow && col == targetCol) continue;
+
+            int cx0 = (int)(col * cellW) + border;
+            int cy0 = (int)(row * cellH) + border;
+            int cInnerW = Math.Min(innerW, imgW - cx0);
+            int cInnerH = Math.Min(innerH, imgH - cy0);
+            if (cInnerW < innerW || cInnerH < innerH) continue;
+
+            // Compute NCC at full resolution
+            double cMean = 0;
+            for (int py = 0; py < innerH; py++)
+                for (int px = 0; px < innerW; px++)
+                    cMean += gray[(cy0 + py) * imgW + (cx0 + px)];
+            cMean /= n;
+
+            double cStd = 0, cross = 0;
+            for (int py = 0; py < innerH; py++)
+                for (int px = 0; px < innerW; px++)
+                {
+                    double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean;
+                    double tv = targetPixels[py * innerW + px] - tMean;
+                    cStd += cv * cv;
+                    cross += tv * cv;
+                }
+            cStd = Math.Sqrt(cStd / n);
+
+            double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0;
+
+            if (debug && ncc > 0.5)
+                Console.Error.WriteLine($"    ({row},{col}): NCC={ncc:F3}");
+
+            if (ncc >= matchThreshold)
+                matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) });
+        }
+    }
+
+    if (debug) Console.Error.WriteLine($"  Matches for ({targetRow},{targetCol}): {matches.Count}");
+    return matches;
 }
 
 void HandleDetectGrid(Request req)
@@ -788,6 +1004,12 @@ class Request
 
     [JsonPropertyName("debug")]
     public bool Debug { get; set; }
+
+    [JsonPropertyName("targetRow")]
+    public int TargetRow { get; set; } = -1;
+
+    [JsonPropertyName("targetCol")]
+    public int TargetCol { get; set; } = -1;
 }
 
 class RegionRect
@@ -884,6 +1106,39 @@ class GridResponse
 
     [JsonPropertyName("cells")]
     public List<List<bool>> Cells { get; set; } = [];
+
+    [JsonPropertyName("items")]
+    public List<GridItem>? Items { get; set; }
+
+    [JsonPropertyName("matches")]
+    public List<GridMatch>? Matches { get; set; }
+}
+
+class GridItem
+{
+    [JsonPropertyName("row")]
+    public int Row { get; set; }
+
+    [JsonPropertyName("col")]
+    public int Col { get; set; }
+
+    [JsonPropertyName("w")]
+    public int W { get; set; }
+
+    [JsonPropertyName("h")]
+    public int H { get; set; }
+}
+
+class GridMatch
+{
+    [JsonPropertyName("row")]
+    public int Row { get; set; }
+
+    [JsonPropertyName("col")]
+    public int Col { get; set; }
+
+    [JsonPropertyName("similarity")]
+    public double Similarity { get; set; }
 }
 
 class DetectGridResponse