finished item finder
This commit is contained in:
parent
1246884be9
commit
930e00c9cc
7 changed files with 450 additions and 37 deletions
|
|
@ -4,6 +4,7 @@ import { WebSocketServer, WebSocket } from 'ws';
|
|||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { logger } from '../util/logger.js';
|
||||
import { sleep } from '../util/sleep.js';
|
||||
import type { BotController } from './BotController.js';
|
||||
import type { ScreenReader } from '../game/ScreenReader.js';
|
||||
import { GRID_LAYOUTS } from '../game/GridReader.js';
|
||||
|
|
@ -203,18 +204,21 @@ export class DashboardServer {
|
|||
// Grid scan with calibrated positions
|
||||
this.app.post('/api/debug/grid-scan', async (req, res) => {
|
||||
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
|
||||
const { layout: layoutName } = req.body as { layout: string };
|
||||
const { layout: layoutName, targetRow, targetCol } = req.body as { layout: string; targetRow?: number; targetCol?: number };
|
||||
if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown grid layout: ${layoutName}` }); return; }
|
||||
try {
|
||||
const result = await this.debug.screenReader.grid.scan(layoutName);
|
||||
const result = await this.debug.screenReader.grid.scan(layoutName, undefined, targetRow, targetCol);
|
||||
const imageBuffer = await this.debug.screenReader.captureRegion(result.layout.region);
|
||||
const imageBase64 = imageBuffer.toString('base64');
|
||||
const r = result.layout.region;
|
||||
const matchInfo = result.matches ? `, ${result.matches.length} matches` : '';
|
||||
this.broadcastLog('info',
|
||||
`Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height} — ${result.occupied.length} occupied cells`);
|
||||
`Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height} — ${result.occupied.length} occupied cells${matchInfo}`);
|
||||
res.json({
|
||||
ok: true,
|
||||
occupied: result.occupied,
|
||||
items: result.items,
|
||||
matches: result.matches,
|
||||
cols: result.layout.cols,
|
||||
rows: result.layout.rows,
|
||||
image: imageBase64,
|
||||
|
|
@ -247,6 +251,51 @@ export class DashboardServer {
|
|||
}
|
||||
});
|
||||
|
||||
// Test: scan grid, find matches for target cell, hover over each for 1s
|
||||
this.app.post('/api/debug/test-match-hover', async (req, res) => {
|
||||
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
|
||||
const { layout: layoutName, targetRow, targetCol } = req.body as { layout: string; targetRow: number; targetCol: number };
|
||||
if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown layout: ${layoutName}` }); return; }
|
||||
if (targetRow == null || targetCol == null) { res.status(400).json({ error: 'Missing targetRow/targetCol' }); return; }
|
||||
try {
|
||||
// Scan with match target
|
||||
this.broadcastLog('info', `Scanning ${layoutName} with target (${targetRow},${targetCol})...`);
|
||||
const result = await this.debug.screenReader.grid.scan(layoutName, undefined, targetRow, targetCol);
|
||||
const matches = result.matches ?? [];
|
||||
const items = result.items ?? [];
|
||||
|
||||
// Find the item dimensions at target cell
|
||||
const targetItem = items.find(i => targetRow >= i.row && targetRow < i.row + i.h && targetCol >= i.col && targetCol < i.col + i.w);
|
||||
const itemSize = targetItem ? `${targetItem.w}x${targetItem.h}` : '1x1';
|
||||
this.broadcastLog('info', `Target (${targetRow},${targetCol}) is ${itemSize}, found ${matches.length} matches`);
|
||||
|
||||
// Build list of cells to hover: target first, then matches
|
||||
const hoverCells = [
|
||||
{ row: targetRow, col: targetCol, label: 'TARGET' },
|
||||
...matches.map(m => ({ row: m.row, col: m.col, label: `MATCH ${(m.similarity * 100).toFixed(0)}%` })),
|
||||
];
|
||||
|
||||
// Focus game and hover each cell
|
||||
await this.debug.gameController.focusGame();
|
||||
for (const cell of hoverCells) {
|
||||
const center = result.layout.region;
|
||||
const cellW = center.width / result.layout.cols;
|
||||
const cellH = center.height / result.layout.rows;
|
||||
const x = Math.round(center.x + cell.col * cellW + cellW / 2);
|
||||
const y = Math.round(center.y + cell.row * cellH + cellH / 2);
|
||||
this.broadcastLog('info', `Hovering ${cell.label} (${cell.row},${cell.col}) at (${x},${y})...`);
|
||||
await this.debug.gameController.moveMouseTo(x, y);
|
||||
await sleep(1000);
|
||||
}
|
||||
|
||||
this.broadcastLog('info', `Done — hovered ${hoverCells.length} cells`);
|
||||
res.json({ ok: true, itemSize, matchCount: matches.length, hoveredCount: hoverCells.length });
|
||||
} catch (err) {
|
||||
logger.error({ err }, 'Debug test-match-hover failed');
|
||||
res.status(500).json({ error: 'Test match hover failed' });
|
||||
}
|
||||
});
|
||||
|
||||
this.server = http.createServer(this.app);
|
||||
this.wss = new WebSocketServer({ server: this.server });
|
||||
|
||||
|
|
|
|||
|
|
@ -397,6 +397,19 @@
|
|||
<button onclick="debugGridScan('shop')">Scan Shop</button>
|
||||
<button onclick="debugGridScan('vendor')">Scan Vendor</button>
|
||||
</div>
|
||||
<div class="debug-row">
|
||||
<select id="matchLayout" style="padding:6px 10px;background:#0d1117;border:1px solid #30363d;border-radius:6px;color:#e6edf3;font-size:13px">
|
||||
<option value="shop">Shop</option>
|
||||
<option value="seller">Seller</option>
|
||||
<option value="stash12">Stash 12×12</option>
|
||||
<option value="stash12_folder">Stash 12×12 (folder)</option>
|
||||
<option value="inventory">Inventory</option>
|
||||
<option value="vendor">Vendor</option>
|
||||
</select>
|
||||
<input type="number" id="matchRow" placeholder="Row" value="8" style="width:60px" />
|
||||
<input type="number" id="matchCol" placeholder="Col" value="0" style="width:60px" />
|
||||
<button class="primary" onclick="debugTestMatchHover()">Match & Hover</button>
|
||||
</div>
|
||||
<div class="debug-row">
|
||||
<input type="text" id="debugTextInput" placeholder="Text to find (e.g. Stash, Ange)" />
|
||||
<button onclick="debugFindText()">Find</button>
|
||||
|
|
@ -645,33 +658,81 @@
|
|||
}
|
||||
|
||||
// Debug functions
|
||||
async function debugGridScan(layout) {
|
||||
showDebugResult(`Scanning ${layout}...`);
|
||||
let lastGridLayout = null;
|
||||
|
||||
async function debugGridScan(layout, targetRow, targetCol) {
|
||||
lastGridLayout = layout;
|
||||
const hasTarget = targetRow != null && targetCol != null;
|
||||
showDebugResult(hasTarget ? `Matching (${targetRow},${targetCol}) in ${layout}...` : `Scanning ${layout}...`);
|
||||
const body = { layout };
|
||||
if (hasTarget) { body.targetRow = targetRow; body.targetCol = targetCol; }
|
||||
const res = await fetch('/api/debug/grid-scan', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ layout }),
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
|
||||
const el = document.getElementById('debugResult');
|
||||
const count = data.occupied.length;
|
||||
const items = data.items || [];
|
||||
const matches = data.matches || [];
|
||||
const r = data.region;
|
||||
let html = `<b>${layout}</b> ${data.cols}x${data.rows}`;
|
||||
html += ` — ${count} occupied cell(s)`;
|
||||
html += ` — ${count} occupied, ${items.length} items`;
|
||||
if (matches.length > 0) html += `, <span style="color:#f0883e;font-weight:bold">${matches.length} matches</span>`;
|
||||
if (r) html += `<br><span style="color:#484f58">Region: (${r.x}, ${r.y}) ${r.width}x${r.height}</span>`;
|
||||
if (count > 0) {
|
||||
html += `<br>` + data.occupied.map(c => `(${c.row},${c.col})`).join(' ');
|
||||
if (items.length > 0) {
|
||||
const sizes = {};
|
||||
items.forEach(i => { const k = i.w + 'x' + i.h; sizes[k] = (sizes[k]||0) + 1; });
|
||||
html += `<br><span style="color:#58a6ff">` + Object.entries(sizes).map(([k,v]) => `${v}x ${k}`).join(', ') + `</span>`;
|
||||
}
|
||||
if (hasTarget) {
|
||||
html += `<br><span style="color:#f0883e">Target: (${targetRow},${targetCol})`;
|
||||
if (matches.length > 0) html += ` → ${matches.map(m => `(${m.row},${m.col}) ${(m.similarity*100).toFixed(0)}%`).join(', ')}`;
|
||||
html += `</span>`;
|
||||
} else {
|
||||
html += `<br><span style="color:#484f58">Click a cell to find matching items</span>`;
|
||||
}
|
||||
html += '<div class="grid-debug">';
|
||||
if (data.image) {
|
||||
html += `<img src="data:image/png;base64,${data.image}" alt="Grid capture" />`;
|
||||
}
|
||||
// Build match set for highlighting
|
||||
const matchSet = new Set(matches.map(m => m.row + ',' + m.col));
|
||||
const targetKey = hasTarget ? targetRow + ',' + targetCol : null;
|
||||
// Build item map: cell → item info
|
||||
const itemMap = {};
|
||||
const colors = ['#238636','#1f6feb','#8957e5','#da3633','#d29922','#3fb950','#388bfd','#a371f7','#f85149','#e3b341'];
|
||||
items.forEach((item, idx) => {
|
||||
const color = colors[idx % colors.length];
|
||||
for (let dr = 0; dr < item.h; dr++)
|
||||
for (let dc = 0; dc < item.w; dc++)
|
||||
itemMap[(item.row+dr)+','+(item.col+dc)] = { item, color, isOrigin: dr===0 && dc===0 };
|
||||
});
|
||||
html += `<div class="grid-view" style="grid-template-columns:repeat(${data.cols},12px)">`;
|
||||
const set = new Set(data.occupied.map(c => c.row + ',' + c.col));
|
||||
for (let r = 0; r < data.rows; r++) {
|
||||
for (let c = 0; c < data.cols; c++) {
|
||||
html += `<div class="grid-cell${set.has(r+','+c) ? ' occupied' : ''}" title="(${r},${c})"></div>`;
|
||||
const key = r+','+c;
|
||||
const isTarget = key === targetKey;
|
||||
const isMatch = matchSet.has(key);
|
||||
const info = itemMap[key];
|
||||
let bg;
|
||||
if (isTarget) bg = '#f0883e';
|
||||
else if (isMatch) bg = '#d29922';
|
||||
else if (info) bg = info.color;
|
||||
else if (set.has(key)) bg = '#238636';
|
||||
else bg = '';
|
||||
const outline = (isTarget || isMatch) ? 'outline:2px solid #f0883e;z-index:1;' : '';
|
||||
const cursor = set.has(key) ? 'cursor:pointer;' : '';
|
||||
const bgStyle = bg ? `background:${bg};` : '';
|
||||
const style = (bgStyle || outline || cursor) ? ` style="${bgStyle}${outline}${cursor}"` : '';
|
||||
let title = info ? `(${r},${c}) ${info.item.w}x${info.item.h}` : `(${r},${c})`;
|
||||
if (isTarget) title += ' [TARGET]';
|
||||
if (isMatch) { const m = matches.find(m => m.row===r && m.col===c); title += ` [MATCH ${(m.similarity*100).toFixed(0)}%]`; }
|
||||
const onclick = set.has(key) ? ` onclick="debugGridScan('${layout}',${r},${c})"` : '';
|
||||
html += `<div class="grid-cell${set.has(key) ? ' occupied' : ''}"${style}${onclick} title="${title}"></div>`;
|
||||
}
|
||||
}
|
||||
html += '</div></div>';
|
||||
|
|
@ -764,6 +825,22 @@
|
|||
showDebugResult(data.ok ? `Clicked at (${x}, ${y})` : `Error: ${data.error}`);
|
||||
}
|
||||
|
||||
async function debugTestMatchHover() {
|
||||
const layout = document.getElementById('matchLayout').value;
|
||||
const targetRow = parseInt(document.getElementById('matchRow').value);
|
||||
const targetCol = parseInt(document.getElementById('matchCol').value);
|
||||
if (isNaN(targetRow) || isNaN(targetCol)) { showDebugResult('Invalid row/col'); return; }
|
||||
showDebugResult(`Scanning ${layout} and matching (${targetRow},${targetCol})...`);
|
||||
const res = await fetch('/api/debug/test-match-hover', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ layout, targetRow, targetCol }),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
|
||||
showDebugResult(`Done: ${data.itemSize} item, ${data.matchCount} matches, hovered ${data.hoveredCount} cells`);
|
||||
}
|
||||
|
||||
function showDebugResult(text) {
|
||||
document.getElementById('debugResult').textContent = text;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,6 +87,10 @@ export class GameController {
|
|||
await this.inputSender.ctrlRightClick(x, y);
|
||||
}
|
||||
|
||||
async moveMouseTo(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.moveMouse(x, y);
|
||||
}
|
||||
|
||||
async leftClickAt(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.leftClick(x, y);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { logger } from '../util/logger.js';
|
||||
import type { OcrDaemon } from './OcrDaemon.js';
|
||||
import type { OcrDaemon, GridItem, GridMatch } from './OcrDaemon.js';
|
||||
import type { Region } from '../types.js';
|
||||
|
||||
// ── Grid type definitions ───────────────────────────────────────────────────
|
||||
|
|
@ -20,6 +20,8 @@ export interface CellCoord {
|
|||
export interface ScanResult {
|
||||
layout: GridLayout;
|
||||
occupied: CellCoord[];
|
||||
items: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
}
|
||||
|
||||
// ── Calibrated grid layouts (2560×1440) ─────────────────────────────────────
|
||||
|
|
@ -89,20 +91,20 @@ export class GridReader {
|
|||
/**
|
||||
* Scan a named grid layout for occupied cells.
|
||||
*/
|
||||
async scan(layoutName: string, threshold?: number): Promise<ScanResult> {
|
||||
async scan(layoutName: string, threshold?: number, targetRow?: number, targetCol?: number): Promise<ScanResult> {
|
||||
const layout = GRID_LAYOUTS[layoutName];
|
||||
if (!layout) throw new Error(`Unknown grid layout: ${layoutName}`);
|
||||
|
||||
const t = performance.now();
|
||||
const occupied = await this.getOccupiedCells(layout, threshold);
|
||||
const { occupied, items, matches } = await this.getOccupiedCells(layout, threshold, targetRow, targetCol);
|
||||
|
||||
const ms = (performance.now() - t).toFixed(0);
|
||||
logger.info(
|
||||
{ layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, ms },
|
||||
{ layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, items: items.length, matches: matches?.length, ms },
|
||||
'Grid scan complete',
|
||||
);
|
||||
|
||||
return { layout, occupied };
|
||||
return { layout, occupied, items, matches };
|
||||
}
|
||||
|
||||
/** Get the screen-space center of a grid cell */
|
||||
|
|
@ -115,20 +117,22 @@ export class GridReader {
|
|||
};
|
||||
}
|
||||
|
||||
/** Scan the grid and return which cells are occupied */
|
||||
async getOccupiedCells(layout: GridLayout, threshold?: number): Promise<CellCoord[]> {
|
||||
/** Scan the grid and return which cells are occupied and detected items */
|
||||
async getOccupiedCells(layout: GridLayout, threshold?: number, targetRow?: number, targetCol?: number): Promise<{ occupied: CellCoord[]; items: GridItem[]; matches?: GridMatch[] }> {
|
||||
const t = performance.now();
|
||||
const cells = await this.daemon.gridScan(
|
||||
const result = await this.daemon.gridScan(
|
||||
layout.region,
|
||||
layout.cols,
|
||||
layout.rows,
|
||||
threshold,
|
||||
targetRow,
|
||||
targetCol,
|
||||
);
|
||||
|
||||
const occupied: CellCoord[] = [];
|
||||
for (let row = 0; row < cells.length; row++) {
|
||||
for (let col = 0; col < cells[row].length; col++) {
|
||||
if (cells[row][col]) {
|
||||
for (let row = 0; row < result.cells.length; row++) {
|
||||
for (let col = 0; col < result.cells[row].length; col++) {
|
||||
if (result.cells[row][col]) {
|
||||
const center = this.getCellCenter(layout, row, col);
|
||||
occupied.push({ row, col, x: center.x, y: center.y });
|
||||
}
|
||||
|
|
@ -137,10 +141,10 @@ export class GridReader {
|
|||
|
||||
const ms = (performance.now() - t).toFixed(0);
|
||||
logger.info(
|
||||
{ layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, ms },
|
||||
{ layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, items: result.items.length, matches: result.matches?.length, ms },
|
||||
'Grid scan complete',
|
||||
);
|
||||
return occupied;
|
||||
return { occupied, items: result.items, matches: result.matches };
|
||||
}
|
||||
|
||||
/** Get all cell centers in the grid */
|
||||
|
|
|
|||
|
|
@ -24,6 +24,25 @@ export interface OcrResponse {
|
|||
lines: OcrLine[];
|
||||
}
|
||||
|
||||
export interface GridItem {
|
||||
row: number;
|
||||
col: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface GridMatch {
|
||||
row: number;
|
||||
col: number;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
export interface GridScanResult {
|
||||
cells: boolean[][];
|
||||
items: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
}
|
||||
|
||||
export interface DetectGridResult {
|
||||
detected: boolean;
|
||||
region?: Region;
|
||||
|
|
@ -51,6 +70,8 @@ interface DaemonResponse {
|
|||
lines?: OcrLine[];
|
||||
image?: string;
|
||||
cells?: boolean[][];
|
||||
items?: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
detected?: boolean;
|
||||
region?: Region;
|
||||
cols?: number;
|
||||
|
|
@ -106,11 +127,13 @@ export class OcrDaemon {
|
|||
return Buffer.from(resp.image!, 'base64');
|
||||
}
|
||||
|
||||
async gridScan(region: Region, cols: number, rows: number, threshold?: number): Promise<boolean[][]> {
|
||||
async gridScan(region: Region, cols: number, rows: number, threshold?: number, targetRow?: number, targetCol?: number): Promise<GridScanResult> {
|
||||
const req: DaemonRequest = { cmd: 'grid', region, cols, rows };
|
||||
if (threshold) req.threshold = threshold;
|
||||
if (targetRow != null && targetRow >= 0) (req as any).targetRow = targetRow;
|
||||
if (targetCol != null && targetCol >= 0) (req as any).targetCol = targetCol;
|
||||
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
|
||||
return resp.cells ?? [];
|
||||
return { cells: resp.cells ?? [], items: resp.items ?? [], matches: resp.matches ?? undefined };
|
||||
}
|
||||
|
||||
async detectGrid(region: Region, minCellSize?: number, maxCellSize?: number): Promise<DetectGridResult> {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue