finished item finder
This commit is contained in:
parent
1246884be9
commit
930e00c9cc
7 changed files with 450 additions and 37 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -7,6 +7,7 @@ browser-data/
|
|||
debug-screenshots/
|
||||
eng.traineddata
|
||||
.claude/
|
||||
nul
|
||||
|
||||
# OcrDaemon build output
|
||||
tools/OcrDaemon/bin/
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { WebSocketServer, WebSocket } from 'ws';
|
|||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { logger } from '../util/logger.js';
|
||||
import { sleep } from '../util/sleep.js';
|
||||
import type { BotController } from './BotController.js';
|
||||
import type { ScreenReader } from '../game/ScreenReader.js';
|
||||
import { GRID_LAYOUTS } from '../game/GridReader.js';
|
||||
|
|
@ -203,18 +204,21 @@ export class DashboardServer {
|
|||
// Grid scan with calibrated positions
|
||||
this.app.post('/api/debug/grid-scan', async (req, res) => {
|
||||
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
|
||||
const { layout: layoutName } = req.body as { layout: string };
|
||||
const { layout: layoutName, targetRow, targetCol } = req.body as { layout: string; targetRow?: number; targetCol?: number };
|
||||
if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown grid layout: ${layoutName}` }); return; }
|
||||
try {
|
||||
const result = await this.debug.screenReader.grid.scan(layoutName);
|
||||
const result = await this.debug.screenReader.grid.scan(layoutName, undefined, targetRow, targetCol);
|
||||
const imageBuffer = await this.debug.screenReader.captureRegion(result.layout.region);
|
||||
const imageBase64 = imageBuffer.toString('base64');
|
||||
const r = result.layout.region;
|
||||
const matchInfo = result.matches ? `, ${result.matches.length} matches` : '';
|
||||
this.broadcastLog('info',
|
||||
`Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height} — ${result.occupied.length} occupied cells`);
|
||||
`Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height} — ${result.occupied.length} occupied cells${matchInfo}`);
|
||||
res.json({
|
||||
ok: true,
|
||||
occupied: result.occupied,
|
||||
items: result.items,
|
||||
matches: result.matches,
|
||||
cols: result.layout.cols,
|
||||
rows: result.layout.rows,
|
||||
image: imageBase64,
|
||||
|
|
@ -247,6 +251,51 @@ export class DashboardServer {
|
|||
}
|
||||
});
|
||||
|
||||
// Test: scan grid, find matches for target cell, hover over each for 1s
|
||||
this.app.post('/api/debug/test-match-hover', async (req, res) => {
|
||||
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
|
||||
const { layout: layoutName, targetRow, targetCol } = req.body as { layout: string; targetRow: number; targetCol: number };
|
||||
if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown layout: ${layoutName}` }); return; }
|
||||
if (targetRow == null || targetCol == null) { res.status(400).json({ error: 'Missing targetRow/targetCol' }); return; }
|
||||
try {
|
||||
// Scan with match target
|
||||
this.broadcastLog('info', `Scanning ${layoutName} with target (${targetRow},${targetCol})...`);
|
||||
const result = await this.debug.screenReader.grid.scan(layoutName, undefined, targetRow, targetCol);
|
||||
const matches = result.matches ?? [];
|
||||
const items = result.items ?? [];
|
||||
|
||||
// Find the item dimensions at target cell
|
||||
const targetItem = items.find(i => targetRow >= i.row && targetRow < i.row + i.h && targetCol >= i.col && targetCol < i.col + i.w);
|
||||
const itemSize = targetItem ? `${targetItem.w}x${targetItem.h}` : '1x1';
|
||||
this.broadcastLog('info', `Target (${targetRow},${targetCol}) is ${itemSize}, found ${matches.length} matches`);
|
||||
|
||||
// Build list of cells to hover: target first, then matches
|
||||
const hoverCells = [
|
||||
{ row: targetRow, col: targetCol, label: 'TARGET' },
|
||||
...matches.map(m => ({ row: m.row, col: m.col, label: `MATCH ${(m.similarity * 100).toFixed(0)}%` })),
|
||||
];
|
||||
|
||||
// Focus game and hover each cell
|
||||
await this.debug.gameController.focusGame();
|
||||
for (const cell of hoverCells) {
|
||||
const center = result.layout.region;
|
||||
const cellW = center.width / result.layout.cols;
|
||||
const cellH = center.height / result.layout.rows;
|
||||
const x = Math.round(center.x + cell.col * cellW + cellW / 2);
|
||||
const y = Math.round(center.y + cell.row * cellH + cellH / 2);
|
||||
this.broadcastLog('info', `Hovering ${cell.label} (${cell.row},${cell.col}) at (${x},${y})...`);
|
||||
await this.debug.gameController.moveMouseTo(x, y);
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
this.broadcastLog('info', `Done — hovered ${hoverCells.length} cells`);
|
||||
res.json({ ok: true, itemSize, matchCount: matches.length, hoveredCount: hoverCells.length });
|
||||
} catch (err) {
|
||||
logger.error({ err }, 'Debug test-match-hover failed');
|
||||
res.status(500).json({ error: 'Test match hover failed' });
|
||||
}
|
||||
});
|
||||
|
||||
this.server = http.createServer(this.app);
|
||||
this.wss = new WebSocketServer({ server: this.server });
|
||||
|
||||
|
|
|
|||
|
|
@ -397,6 +397,19 @@
|
|||
<button onclick="debugGridScan('shop')">Scan Shop</button>
|
||||
<button onclick="debugGridScan('vendor')">Scan Vendor</button>
|
||||
</div>
|
||||
<div class="debug-row">
|
||||
<select id="matchLayout" style="padding:6px 10px;background:#0d1117;border:1px solid #30363d;border-radius:6px;color:#e6edf3;font-size:13px">
|
||||
<option value="shop">Shop</option>
|
||||
<option value="seller">Seller</option>
|
||||
<option value="stash12">Stash 12×12</option>
|
||||
<option value="stash12_folder">Stash 12×12 (folder)</option>
|
||||
<option value="inventory">Inventory</option>
|
||||
<option value="vendor">Vendor</option>
|
||||
</select>
|
||||
<input type="number" id="matchRow" placeholder="Row" value="8" style="width:60px" />
|
||||
<input type="number" id="matchCol" placeholder="Col" value="0" style="width:60px" />
|
||||
<button class="primary" onclick="debugTestMatchHover()">Match & Hover</button>
|
||||
</div>
|
||||
<div class="debug-row">
|
||||
<input type="text" id="debugTextInput" placeholder="Text to find (e.g. Stash, Ange)" />
|
||||
<button onclick="debugFindText()">Find</button>
|
||||
|
|
@ -645,33 +658,81 @@
|
|||
}
|
||||
|
||||
// Debug functions
|
||||
async function debugGridScan(layout) {
|
||||
showDebugResult(`Scanning ${layout}...`);
|
||||
let lastGridLayout = null;
|
||||
|
||||
async function debugGridScan(layout, targetRow, targetCol) {
|
||||
lastGridLayout = layout;
|
||||
const hasTarget = targetRow != null && targetCol != null;
|
||||
showDebugResult(hasTarget ? `Matching (${targetRow},${targetCol}) in ${layout}...` : `Scanning ${layout}...`);
|
||||
const body = { layout };
|
||||
if (hasTarget) { body.targetRow = targetRow; body.targetCol = targetCol; }
|
||||
const res = await fetch('/api/debug/grid-scan', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ layout }),
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
|
||||
const el = document.getElementById('debugResult');
|
||||
const count = data.occupied.length;
|
||||
const items = data.items || [];
|
||||
const matches = data.matches || [];
|
||||
const r = data.region;
|
||||
let html = `<b>${layout}</b> ${data.cols}x${data.rows}`;
|
||||
html += ` — ${count} occupied cell(s)`;
|
||||
html += ` — ${count} occupied, ${items.length} items`;
|
||||
if (matches.length > 0) html += `, <span style="color:#f0883e;font-weight:bold">${matches.length} matches</span>`;
|
||||
if (r) html += `<br><span style="color:#484f58">Region: (${r.x}, ${r.y}) ${r.width}x${r.height}</span>`;
|
||||
if (count > 0) {
|
||||
html += `<br>` + data.occupied.map(c => `(${c.row},${c.col})`).join(' ');
|
||||
if (items.length > 0) {
|
||||
const sizes = {};
|
||||
items.forEach(i => { const k = i.w + 'x' + i.h; sizes[k] = (sizes[k]||0) + 1; });
|
||||
html += `<br><span style="color:#58a6ff">` + Object.entries(sizes).map(([k,v]) => `${v}x ${k}`).join(', ') + `</span>`;
|
||||
}
|
||||
if (hasTarget) {
|
||||
html += `<br><span style="color:#f0883e">Target: (${targetRow},${targetCol})`;
|
||||
if (matches.length > 0) html += ` → ${matches.map(m => `(${m.row},${m.col}) ${(m.similarity*100).toFixed(0)}%`).join(', ')}`;
|
||||
html += `</span>`;
|
||||
} else {
|
||||
html += `<br><span style="color:#484f58">Click a cell to find matching items</span>`;
|
||||
}
|
||||
html += '<div class="grid-debug">';
|
||||
if (data.image) {
|
||||
html += `<img src="data:image/png;base64,${data.image}" alt="Grid capture" />`;
|
||||
}
|
||||
// Build match set for highlighting
|
||||
const matchSet = new Set(matches.map(m => m.row + ',' + m.col));
|
||||
const targetKey = hasTarget ? targetRow + ',' + targetCol : null;
|
||||
// Build item map: cell → item info
|
||||
const itemMap = {};
|
||||
const colors = ['#238636','#1f6feb','#8957e5','#da3633','#d29922','#3fb950','#388bfd','#a371f7','#f85149','#e3b341'];
|
||||
items.forEach((item, idx) => {
|
||||
const color = colors[idx % colors.length];
|
||||
for (let dr = 0; dr < item.h; dr++)
|
||||
for (let dc = 0; dc < item.w; dc++)
|
||||
itemMap[(item.row+dr)+','+(item.col+dc)] = { item, color, isOrigin: dr===0 && dc===0 };
|
||||
});
|
||||
html += `<div class="grid-view" style="grid-template-columns:repeat(${data.cols},12px)">`;
|
||||
const set = new Set(data.occupied.map(c => c.row + ',' + c.col));
|
||||
for (let r = 0; r < data.rows; r++) {
|
||||
for (let c = 0; c < data.cols; c++) {
|
||||
html += `<div class="grid-cell${set.has(r+','+c) ? ' occupied' : ''}" title="(${r},${c})"></div>`;
|
||||
const key = r+','+c;
|
||||
const isTarget = key === targetKey;
|
||||
const isMatch = matchSet.has(key);
|
||||
const info = itemMap[key];
|
||||
let bg;
|
||||
if (isTarget) bg = '#f0883e';
|
||||
else if (isMatch) bg = '#d29922';
|
||||
else if (info) bg = info.color;
|
||||
else if (set.has(key)) bg = '#238636';
|
||||
else bg = '';
|
||||
const outline = (isTarget || isMatch) ? 'outline:2px solid #f0883e;z-index:1;' : '';
|
||||
const cursor = set.has(key) ? 'cursor:pointer;' : '';
|
||||
const bgStyle = bg ? `background:${bg};` : '';
|
||||
const style = (bgStyle || outline || cursor) ? ` style="${bgStyle}${outline}${cursor}"` : '';
|
||||
let title = info ? `(${r},${c}) ${info.item.w}x${info.item.h}` : `(${r},${c})`;
|
||||
if (isTarget) title += ' [TARGET]';
|
||||
if (isMatch) { const m = matches.find(m => m.row===r && m.col===c); title += ` [MATCH ${(m.similarity*100).toFixed(0)}%]`; }
|
||||
const onclick = set.has(key) ? ` onclick="debugGridScan('${layout}',${r},${c})"` : '';
|
||||
html += `<div class="grid-cell${set.has(key) ? ' occupied' : ''}"${style}${onclick} title="${title}"></div>`;
|
||||
}
|
||||
}
|
||||
html += '</div></div>';
|
||||
|
|
@ -764,6 +825,22 @@
|
|||
showDebugResult(data.ok ? `Clicked at (${x}, ${y})` : `Error: ${data.error}`);
|
||||
}
|
||||
|
||||
async function debugTestMatchHover() {
|
||||
const layout = document.getElementById('matchLayout').value;
|
||||
const targetRow = parseInt(document.getElementById('matchRow').value);
|
||||
const targetCol = parseInt(document.getElementById('matchCol').value);
|
||||
if (isNaN(targetRow) || isNaN(targetCol)) { showDebugResult('Invalid row/col'); return; }
|
||||
showDebugResult(`Scanning ${layout} and matching (${targetRow},${targetCol})...`);
|
||||
const res = await fetch('/api/debug/test-match-hover', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ layout, targetRow, targetCol }),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
|
||||
showDebugResult(`Done: ${data.itemSize} item, ${data.matchCount} matches, hovered ${data.hoveredCount} cells`);
|
||||
}
|
||||
|
||||
function showDebugResult(text) {
|
||||
document.getElementById('debugResult').textContent = text;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,6 +87,10 @@ export class GameController {
|
|||
await this.inputSender.ctrlRightClick(x, y);
|
||||
}
|
||||
|
||||
async moveMouseTo(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.moveMouse(x, y);
|
||||
}
|
||||
|
||||
async leftClickAt(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.leftClick(x, y);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { logger } from '../util/logger.js';
|
||||
import type { OcrDaemon } from './OcrDaemon.js';
|
||||
import type { OcrDaemon, GridItem, GridMatch } from './OcrDaemon.js';
|
||||
import type { Region } from '../types.js';
|
||||
|
||||
// ── Grid type definitions ───────────────────────────────────────────────────
|
||||
|
|
@ -20,6 +20,8 @@ export interface CellCoord {
|
|||
export interface ScanResult {
|
||||
layout: GridLayout;
|
||||
occupied: CellCoord[];
|
||||
items: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
}
|
||||
|
||||
// ── Calibrated grid layouts (2560×1440) ─────────────────────────────────────
|
||||
|
|
@ -89,20 +91,20 @@ export class GridReader {
|
|||
/**
|
||||
* Scan a named grid layout for occupied cells.
|
||||
*/
|
||||
async scan(layoutName: string, threshold?: number): Promise<ScanResult> {
|
||||
async scan(layoutName: string, threshold?: number, targetRow?: number, targetCol?: number): Promise<ScanResult> {
|
||||
const layout = GRID_LAYOUTS[layoutName];
|
||||
if (!layout) throw new Error(`Unknown grid layout: ${layoutName}`);
|
||||
|
||||
const t = performance.now();
|
||||
const occupied = await this.getOccupiedCells(layout, threshold);
|
||||
const { occupied, items, matches } = await this.getOccupiedCells(layout, threshold, targetRow, targetCol);
|
||||
|
||||
const ms = (performance.now() - t).toFixed(0);
|
||||
logger.info(
|
||||
{ layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, ms },
|
||||
{ layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, items: items.length, matches: matches?.length, ms },
|
||||
'Grid scan complete',
|
||||
);
|
||||
|
||||
return { layout, occupied };
|
||||
return { layout, occupied, items, matches };
|
||||
}
|
||||
|
||||
/** Get the screen-space center of a grid cell */
|
||||
|
|
@ -115,20 +117,22 @@ export class GridReader {
|
|||
};
|
||||
}
|
||||
|
||||
/** Scan the grid and return which cells are occupied */
|
||||
async getOccupiedCells(layout: GridLayout, threshold?: number): Promise<CellCoord[]> {
|
||||
/** Scan the grid and return which cells are occupied and detected items */
|
||||
async getOccupiedCells(layout: GridLayout, threshold?: number, targetRow?: number, targetCol?: number): Promise<{ occupied: CellCoord[]; items: GridItem[]; matches?: GridMatch[] }> {
|
||||
const t = performance.now();
|
||||
const cells = await this.daemon.gridScan(
|
||||
const result = await this.daemon.gridScan(
|
||||
layout.region,
|
||||
layout.cols,
|
||||
layout.rows,
|
||||
threshold,
|
||||
targetRow,
|
||||
targetCol,
|
||||
);
|
||||
|
||||
const occupied: CellCoord[] = [];
|
||||
for (let row = 0; row < cells.length; row++) {
|
||||
for (let col = 0; col < cells[row].length; col++) {
|
||||
if (cells[row][col]) {
|
||||
for (let row = 0; row < result.cells.length; row++) {
|
||||
for (let col = 0; col < result.cells[row].length; col++) {
|
||||
if (result.cells[row][col]) {
|
||||
const center = this.getCellCenter(layout, row, col);
|
||||
occupied.push({ row, col, x: center.x, y: center.y });
|
||||
}
|
||||
|
|
@ -137,10 +141,10 @@ export class GridReader {
|
|||
|
||||
const ms = (performance.now() - t).toFixed(0);
|
||||
logger.info(
|
||||
{ layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, ms },
|
||||
{ layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, items: result.items.length, matches: result.matches?.length, ms },
|
||||
'Grid scan complete',
|
||||
);
|
||||
return occupied;
|
||||
return { occupied, items: result.items, matches: result.matches };
|
||||
}
|
||||
|
||||
/** Get all cell centers in the grid */
|
||||
|
|
|
|||
|
|
@ -24,6 +24,25 @@ export interface OcrResponse {
|
|||
lines: OcrLine[];
|
||||
}
|
||||
|
||||
export interface GridItem {
|
||||
row: number;
|
||||
col: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface GridMatch {
|
||||
row: number;
|
||||
col: number;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
export interface GridScanResult {
|
||||
cells: boolean[][];
|
||||
items: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
}
|
||||
|
||||
export interface DetectGridResult {
|
||||
detected: boolean;
|
||||
region?: Region;
|
||||
|
|
@ -51,6 +70,8 @@ interface DaemonResponse {
|
|||
lines?: OcrLine[];
|
||||
image?: string;
|
||||
cells?: boolean[][];
|
||||
items?: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
detected?: boolean;
|
||||
region?: Region;
|
||||
cols?: number;
|
||||
|
|
@ -106,11 +127,13 @@ export class OcrDaemon {
|
|||
return Buffer.from(resp.image!, 'base64');
|
||||
}
|
||||
|
||||
async gridScan(region: Region, cols: number, rows: number, threshold?: number): Promise<boolean[][]> {
|
||||
async gridScan(region: Region, cols: number, rows: number, threshold?: number, targetRow?: number, targetCol?: number): Promise<GridScanResult> {
|
||||
const req: DaemonRequest = { cmd: 'grid', region, cols, rows };
|
||||
if (threshold) req.threshold = threshold;
|
||||
if (targetRow != null && targetRow >= 0) (req as any).targetRow = targetRow;
|
||||
if (targetCol != null && targetCol >= 0) (req as any).targetCol = targetCol;
|
||||
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
|
||||
return resp.cells ?? [];
|
||||
return { cells: resp.cells ?? [], items: resp.items ?? [], matches: resp.matches ?? undefined };
|
||||
}
|
||||
|
||||
async detectGrid(region: Region, minCellSize?: number, maxCellSize?: number): Promise<DetectGridResult> {
|
||||
|
|
|
|||
|
|
@ -129,10 +129,13 @@ void HandleCapture(Request req)
|
|||
}
|
||||
|
||||
// Pre-loaded empty cell templates (loaded lazily on first grid scan)
|
||||
// Stored as both grayscale (for occupied detection) and ARGB (for item border detection)
|
||||
byte[]? emptyTemplate70Gray = null;
|
||||
int emptyTemplate70W = 0, emptyTemplate70H = 0;
|
||||
byte[]? emptyTemplate70Argb = null;
|
||||
int emptyTemplate70W = 0, emptyTemplate70H = 0, emptyTemplate70Stride = 0;
|
||||
byte[]? emptyTemplate35Gray = null;
|
||||
int emptyTemplate35W = 0, emptyTemplate35H = 0;
|
||||
byte[]? emptyTemplate35Argb = null;
|
||||
int emptyTemplate35W = 0, emptyTemplate35H = 0, emptyTemplate35Stride = 0;
|
||||
|
||||
void LoadTemplatesIfNeeded()
|
||||
{
|
||||
|
|
@ -150,23 +153,23 @@ void LoadTemplatesIfNeeded()
|
|||
using var bmp = new Bitmap(t70Path);
|
||||
emptyTemplate70W = bmp.Width;
|
||||
emptyTemplate70H = bmp.Height;
|
||||
emptyTemplate70Gray = BitmapToGray(bmp);
|
||||
(emptyTemplate70Gray, emptyTemplate70Argb, emptyTemplate70Stride) = BitmapToGrayAndArgb(bmp);
|
||||
}
|
||||
if (System.IO.File.Exists(t35Path))
|
||||
{
|
||||
using var bmp = new Bitmap(t35Path);
|
||||
emptyTemplate35W = bmp.Width;
|
||||
emptyTemplate35H = bmp.Height;
|
||||
emptyTemplate35Gray = BitmapToGray(bmp);
|
||||
(emptyTemplate35Gray, emptyTemplate35Argb, emptyTemplate35Stride) = BitmapToGrayAndArgb(bmp);
|
||||
}
|
||||
}
|
||||
|
||||
byte[] BitmapToGray(Bitmap bmp)
|
||||
(byte[] gray, byte[] argb, int stride) BitmapToGrayAndArgb(Bitmap bmp)
|
||||
{
|
||||
int w = bmp.Width, h = bmp.Height;
|
||||
var data = bmp.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
|
||||
byte[] pixels = new byte[data.Stride * h];
|
||||
Marshal.Copy(data.Scan0, pixels, 0, pixels.Length);
|
||||
byte[] argb = new byte[data.Stride * h];
|
||||
Marshal.Copy(data.Scan0, argb, 0, argb.Length);
|
||||
bmp.UnlockBits(data);
|
||||
int stride = data.Stride;
|
||||
|
||||
|
|
@ -175,8 +178,14 @@ byte[] BitmapToGray(Bitmap bmp)
|
|||
for (int x = 0; x < w; x++)
|
||||
{
|
||||
int i = y * stride + x * 4;
|
||||
gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
|
||||
gray[y * w + x] = (byte)((argb[i] + argb[i + 1] + argb[i + 2]) / 3);
|
||||
}
|
||||
return (gray, argb, stride);
|
||||
}
|
||||
|
||||
byte[] BitmapToGray(Bitmap bmp)
|
||||
{
|
||||
var (gray, _, _) = BitmapToGrayAndArgb(bmp);
|
||||
return gray;
|
||||
}
|
||||
|
||||
|
|
@ -199,18 +208,23 @@ void HandleGrid(Request req)
|
|||
// Pick the right empty template based on cell size
|
||||
int nominalCell = (int)Math.Round(cellW);
|
||||
byte[]? templateGray;
|
||||
int templateW, templateH;
|
||||
byte[]? templateArgb;
|
||||
int templateW, templateH, templateStride;
|
||||
if (nominalCell <= 40 && emptyTemplate35Gray != null)
|
||||
{
|
||||
templateGray = emptyTemplate35Gray;
|
||||
templateArgb = emptyTemplate35Argb!;
|
||||
templateW = emptyTemplate35W;
|
||||
templateH = emptyTemplate35H;
|
||||
templateStride = emptyTemplate35Stride;
|
||||
}
|
||||
else if (emptyTemplate70Gray != null)
|
||||
{
|
||||
templateGray = emptyTemplate70Gray;
|
||||
templateArgb = emptyTemplate70Argb!;
|
||||
templateW = emptyTemplate70W;
|
||||
templateH = emptyTemplate70H;
|
||||
templateStride = emptyTemplate70Stride;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -218,8 +232,8 @@ void HandleGrid(Request req)
|
|||
return;
|
||||
}
|
||||
|
||||
// Convert captured bitmap to grayscale
|
||||
byte[] captureGray = BitmapToGray(bitmap);
|
||||
// Convert captured bitmap to grayscale + keep ARGB for border color comparison
|
||||
var (captureGray, captureArgb, captureStride) = BitmapToGrayAndArgb(bitmap);
|
||||
int captureW = bitmap.Width;
|
||||
|
||||
// Border to skip (outer pixels may differ between cells)
|
||||
|
|
@ -277,7 +291,209 @@ void HandleGrid(Request req)
|
|||
if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}");
|
||||
}
|
||||
|
||||
WriteResponse(new GridResponse { Cells = cells });
|
||||
// ── Item detection: compare border pixels to empty template (grayscale) ──
|
||||
// Items have a colored tint behind them that shows through grid lines.
|
||||
// Compare each cell's border strip against the template's border pixels.
|
||||
// If they differ → item tint present → cells belong to same item.
|
||||
int[] parent = new int[rows * cols];
|
||||
for (int i = 0; i < parent.Length; i++) parent[i] = i;
|
||||
|
||||
int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; }
|
||||
void Union(int a, int b) { parent[Find(a)] = Find(b); }
|
||||
|
||||
int stripWidth = Math.Max(2, border / 2);
|
||||
int stripInset = (int)(cellW * 0.15);
|
||||
double borderDiffThresh = 15.0;
|
||||
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
if (!cells[row][col]) continue;
|
||||
int cx0 = (int)(col * cellW);
|
||||
int cy0 = (int)(row * cellH);
|
||||
|
||||
// Check right neighbor
|
||||
if (col + 1 < cols && cells[row][col + 1])
|
||||
{
|
||||
long diffSum = 0; int cnt = 0;
|
||||
int xStart = (int)((col + 1) * cellW) - stripWidth;
|
||||
int yFrom = cy0 + stripInset;
|
||||
int yTo = (int)((row + 1) * cellH) - stripInset;
|
||||
for (int sy = yFrom; sy < yTo; sy += 2)
|
||||
{
|
||||
int tmplY = sy - cy0;
|
||||
for (int sx = xStart; sx < xStart + stripWidth * 2; sx++)
|
||||
{
|
||||
if (sx < 0 || sx >= captureW) continue;
|
||||
int tmplX = sx - cx0;
|
||||
if (tmplX < 0 || tmplX >= templateW) continue;
|
||||
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
||||
if (debug) Console.Error.WriteLine($" H ({row},{col})->({row},{col+1}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
||||
if (meanDiff > borderDiffThresh)
|
||||
Union(row * cols + col, row * cols + col + 1);
|
||||
}
|
||||
|
||||
// Check bottom neighbor
|
||||
if (row + 1 < rows && cells[row + 1][col])
|
||||
{
|
||||
long diffSum = 0; int cnt = 0;
|
||||
int yStart = (int)((row + 1) * cellH) - stripWidth;
|
||||
int xFrom = cx0 + stripInset;
|
||||
int xTo = (int)((col + 1) * cellW) - stripInset;
|
||||
for (int sx = xFrom; sx < xTo; sx += 2)
|
||||
{
|
||||
int tmplX = sx - cx0;
|
||||
for (int sy = yStart; sy < yStart + stripWidth * 2; sy++)
|
||||
{
|
||||
if (sy < 0 || sy >= bitmap.Height) continue;
|
||||
int tmplY = sy - cy0;
|
||||
if (tmplY < 0 || tmplY >= templateH) continue;
|
||||
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
||||
if (debug) Console.Error.WriteLine($" V ({row},{col})->({row+1},{col}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
||||
if (meanDiff > borderDiffThresh)
|
||||
Union(row * cols + col, (row + 1) * cols + col);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract items from union-find groups
|
||||
var groups = new Dictionary<int, List<(int row, int col)>>();
|
||||
for (int row = 0; row < rows; row++)
|
||||
for (int col = 0; col < cols; col++)
|
||||
if (cells[row][col])
|
||||
{
|
||||
int root = Find(row * cols + col);
|
||||
if (!groups.ContainsKey(root)) groups[root] = [];
|
||||
groups[root].Add((row, col));
|
||||
}
|
||||
|
||||
var items = new List<GridItem>();
|
||||
foreach (var group in groups.Values)
|
||||
{
|
||||
int minR = group.Min(c => c.row);
|
||||
int maxR = group.Max(c => c.row);
|
||||
int minC = group.Min(c => c.col);
|
||||
int maxC = group.Max(c => c.col);
|
||||
items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 });
|
||||
}
|
||||
|
||||
if (debug)
|
||||
{
|
||||
Console.Error.WriteLine($" Items found: {items.Count}");
|
||||
foreach (var item in items)
|
||||
Console.Error.WriteLine($" ({item.Row},{item.Col}) {item.W}x{item.H}");
|
||||
}
|
||||
|
||||
// ── Visual matching: find cells similar to target ──
|
||||
List<GridMatch>? matches = null;
|
||||
if (req.TargetRow >= 0 && req.TargetCol >= 0 &&
|
||||
req.TargetRow < rows && req.TargetCol < cols &&
|
||||
cells[req.TargetRow][req.TargetCol])
|
||||
{
|
||||
matches = FindMatchingCells(
|
||||
captureGray, captureW, bitmap.Height,
|
||||
cells, rows, cols, cellW, cellH, border,
|
||||
req.TargetRow, req.TargetCol, debug);
|
||||
}
|
||||
|
||||
WriteResponse(new GridResponse { Cells = cells, Items = items, Matches = matches });
|
||||
}
|
||||
|
||||
/// Find all occupied cells visually similar to the target cell using full-resolution NCC.
|
||||
/// Full resolution gives better discrimination — sockets are a small fraction of total pixels.
|
||||
List<GridMatch> FindMatchingCells(
|
||||
byte[] gray, int imgW, int imgH,
|
||||
List<List<bool>> cells, int rows, int cols,
|
||||
float cellW, float cellH, int border,
|
||||
int targetRow, int targetCol, bool debug)
|
||||
{
|
||||
int innerW = (int)cellW - border * 2;
|
||||
int innerH = (int)cellH - border * 2;
|
||||
if (innerW <= 4 || innerH <= 4) return [];
|
||||
|
||||
int tCx0 = (int)(targetCol * cellW) + border;
|
||||
int tCy0 = (int)(targetRow * cellH) + border;
|
||||
int tInnerW = Math.Min(innerW, imgW - tCx0);
|
||||
int tInnerH = Math.Min(innerH, imgH - tCy0);
|
||||
if (tInnerW < innerW || tInnerH < innerH) return [];
|
||||
|
||||
int n = innerW * innerH;
|
||||
|
||||
// Pre-compute target cell pixels and stats
|
||||
double[] targetPixels = new double[n];
|
||||
double tMean = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
{
|
||||
double v = gray[(tCy0 + py) * imgW + (tCx0 + px)];
|
||||
targetPixels[py * innerW + px] = v;
|
||||
tMean += v;
|
||||
}
|
||||
tMean /= n;
|
||||
|
||||
double tStd = 0;
|
||||
for (int i = 0; i < n; i++)
|
||||
tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean);
|
||||
tStd = Math.Sqrt(tStd / n);
|
||||
|
||||
if (debug) Console.Error.WriteLine($" Match target ({targetRow},{targetCol}): {innerW}x{innerH} ({n}px), mean={tMean:F1}, std={tStd:F1}");
|
||||
if (tStd < 3.0) return [];
|
||||
|
||||
double matchThreshold = 0.70;
|
||||
var matches = new List<GridMatch>();
|
||||
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
if (!cells[row][col]) continue;
|
||||
if (row == targetRow && col == targetCol) continue;
|
||||
|
||||
int cx0 = (int)(col * cellW) + border;
|
||||
int cy0 = (int)(row * cellH) + border;
|
||||
int cInnerW = Math.Min(innerW, imgW - cx0);
|
||||
int cInnerH = Math.Min(innerH, imgH - cy0);
|
||||
if (cInnerW < innerW || cInnerH < innerH) continue;
|
||||
|
||||
// Compute NCC at full resolution
|
||||
double cMean = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
cMean += gray[(cy0 + py) * imgW + (cx0 + px)];
|
||||
cMean /= n;
|
||||
|
||||
double cStd = 0, cross = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
{
|
||||
double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean;
|
||||
double tv = targetPixels[py * innerW + px] - tMean;
|
||||
cStd += cv * cv;
|
||||
cross += tv * cv;
|
||||
}
|
||||
cStd = Math.Sqrt(cStd / n);
|
||||
|
||||
double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0;
|
||||
|
||||
if (debug && ncc > 0.5)
|
||||
Console.Error.WriteLine($" ({row},{col}): NCC={ncc:F3}");
|
||||
|
||||
if (ncc >= matchThreshold)
|
||||
matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) });
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) Console.Error.WriteLine($" Matches for ({targetRow},{targetCol}): {matches.Count}");
|
||||
return matches;
|
||||
}
|
||||
|
||||
void HandleDetectGrid(Request req)
|
||||
|
|
@ -788,6 +1004,12 @@ class Request
|
|||
|
||||
[JsonPropertyName("debug")]
|
||||
public bool Debug { get; set; }
|
||||
|
||||
[JsonPropertyName("targetRow")]
|
||||
public int TargetRow { get; set; } = -1;
|
||||
|
||||
[JsonPropertyName("targetCol")]
|
||||
public int TargetCol { get; set; } = -1;
|
||||
}
|
||||
|
||||
class RegionRect
|
||||
|
|
@ -884,6 +1106,39 @@ class GridResponse
|
|||
|
||||
[JsonPropertyName("cells")]
|
||||
public List<List<bool>> Cells { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("items")]
|
||||
public List<GridItem>? Items { get; set; }
|
||||
|
||||
[JsonPropertyName("matches")]
|
||||
public List<GridMatch>? Matches { get; set; }
|
||||
}
|
||||
|
||||
class GridItem
|
||||
{
|
||||
[JsonPropertyName("row")]
|
||||
public int Row { get; set; }
|
||||
|
||||
[JsonPropertyName("col")]
|
||||
public int Col { get; set; }
|
||||
|
||||
[JsonPropertyName("w")]
|
||||
public int W { get; set; }
|
||||
|
||||
[JsonPropertyName("h")]
|
||||
public int H { get; set; }
|
||||
}
|
||||
|
||||
class GridMatch
|
||||
{
|
||||
[JsonPropertyName("row")]
|
||||
public int Row { get; set; }
|
||||
|
||||
[JsonPropertyName("col")]
|
||||
public int Col { get; set; }
|
||||
|
||||
[JsonPropertyName("similarity")]
|
||||
public double Similarity { get; set; }
|
||||
}
|
||||
|
||||
class DetectGridResponse
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue