finished up grid detection.

This commit is contained in:
Boki 2026-02-10 17:29:37 -05:00
parent 41d174195e
commit 1246884be9
11 changed files with 1037 additions and 19 deletions

View file

@ -6,6 +6,7 @@ import { fileURLToPath } from 'url';
import { logger } from '../util/logger.js';
import type { BotController } from './BotController.js';
import type { ScreenReader } from '../game/ScreenReader.js';
import { GRID_LAYOUTS } from '../game/GridReader.js';
import type { GameController } from '../game/GameController.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@ -150,6 +151,81 @@ export class DashboardServer {
}
});
this.app.post('/api/debug/hideout', async (_req, res) => {
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
try {
await this.debug.gameController.focusGame();
await this.debug.gameController.goToHideout();
this.broadcastLog('info', 'Sent /hideout command');
res.json({ ok: true });
} catch (err) {
logger.error({ err }, 'Debug hideout failed');
res.status(500).json({ error: 'Hideout command failed' });
}
});
// Click first text, then wait for second text to appear and click it
this.app.post('/api/debug/click-then-click', async (req, res) => {
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
const { first, second, timeout = 5000 } = req.body as { first: string; second: string; timeout?: number };
if (!first || !second) { res.status(400).json({ error: 'Missing first/second' }); return; }
try {
// Click the first target
const pos1 = await this.debug.screenReader.findTextOnScreen(first);
if (!pos1) {
this.broadcastLog('warn', `"${first}" not found on screen`);
res.json({ ok: true, found: false, step: 'first' });
return;
}
await this.debug.gameController.focusGame();
await this.debug.gameController.leftClickAt(pos1.x, pos1.y);
this.broadcastLog('info', `Clicked "${first}" at (${pos1.x}, ${pos1.y}), waiting for "${second}"...`);
// Poll OCR until second text appears
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
const pos2 = await this.debug.screenReader.findTextOnScreen(second);
if (pos2) {
await this.debug.gameController.leftClickAt(pos2.x, pos2.y);
this.broadcastLog('info', `Clicked "${second}" at (${pos2.x}, ${pos2.y})`);
res.json({ ok: true, found: true, position: pos2 });
return;
}
}
this.broadcastLog('warn', `"${second}" not found after clicking "${first}" (timed out)`);
res.json({ ok: true, found: false, step: 'second' });
} catch (err) {
logger.error({ err }, 'Debug click-then-click failed');
res.status(500).json({ error: 'Click-then-click failed' });
}
});
// Grid scan with calibrated positions
this.app.post('/api/debug/grid-scan', async (req, res) => {
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
const { layout: layoutName } = req.body as { layout: string };
if (!GRID_LAYOUTS[layoutName]) { res.status(400).json({ error: `Unknown grid layout: ${layoutName}` }); return; }
try {
const result = await this.debug.screenReader.grid.scan(layoutName);
const imageBuffer = await this.debug.screenReader.captureRegion(result.layout.region);
const imageBase64 = imageBuffer.toString('base64');
const r = result.layout.region;
this.broadcastLog('info',
`Grid scan (${layoutName}): ${result.layout.cols}x${result.layout.rows} at (${r.x},${r.y}) ${r.width}x${r.height}${result.occupied.length} occupied cells`);
res.json({
ok: true,
occupied: result.occupied,
cols: result.layout.cols,
rows: result.layout.rows,
image: imageBase64,
region: result.layout.region,
});
} catch (err) {
logger.error({ err }, 'Debug grid-scan failed');
res.status(500).json({ error: 'Grid scan failed' });
}
});
this.app.post('/api/debug/find-and-click', async (req, res) => {
if (!this.debug) { res.status(503).json({ error: 'Debug not available' }); return; }
const { text } = req.body as { text: string };

View file

@ -274,10 +274,48 @@
font-size: 12px;
color: #8b949e;
white-space: pre-wrap;
max-height: 200px;
overflow-y: auto;
}
.debug-result:empty { display: none; }
.grid-debug {
display: flex;
gap: 8px;
margin-top: 8px;
align-items: flex-start;
}
.grid-debug img {
border: 1px solid #30363d;
border-radius: 4px;
max-width: 280px;
max-height: 280px;
object-fit: contain;
}
.grid-view {
display: inline-grid;
gap: 1px;
background: #30363d;
border: 1px solid #30363d;
border-radius: 4px;
overflow: hidden;
flex-shrink: 0;
}
.grid-cell {
width: 12px; height: 12px;
background: #0d1117;
}
.grid-cell.occupied {
background: #238636;
}
.detect-badge {
display: inline-block;
font-size: 10px;
padding: 1px 6px;
border-radius: 4px;
margin-left: 6px;
font-weight: 600;
}
.detect-badge.ok { background: #238636; color: #fff; }
.detect-badge.fallback { background: #9e6a03; color: #fff; }
</style>
</head>
<body>
@ -339,6 +377,25 @@
<div class="debug-row">
<button onclick="debugScreenshot()">Screenshot</button>
<button onclick="debugOcr()">OCR Screen</button>
<button onclick="debugHideout()">Go Hideout</button>
</div>
<div class="debug-row">
<button onclick="debugFindAndClick('ANGE')">ANGE</button>
<button onclick="debugFindAndClick('STASH')">STASH</button>
</div>
<div class="debug-row">
<button onclick="debugAngeOption('Currency')">Currency Exchange</button>
<button onclick="debugAngeOption('Manage')">Manage Shop</button>
<button onclick="debugAngeOption('Buy')">Buy or Sell</button>
<button onclick="debugAngeOption('Purchase')">Purchase Items</button>
</div>
<div class="debug-row">
<button onclick="debugGridScan('inventory')">Scan Inventory</button>
<button onclick="debugGridScan('stash12')">Scan Stash 12×12</button>
<button onclick="debugGridScan('stash24')">Scan Stash 24×24</button>
<button onclick="debugGridScan('seller')">Scan Seller</button>
<button onclick="debugGridScan('shop')">Scan Shop</button>
<button onclick="debugGridScan('vendor')">Scan Vendor</button>
</div>
<div class="debug-row">
<input type="text" id="debugTextInput" placeholder="Text to find (e.g. Stash, Ange)" />
@ -588,6 +645,63 @@
}
// Debug functions
async function debugGridScan(layout) {
showDebugResult(`Scanning ${layout}...`);
const res = await fetch('/api/debug/grid-scan', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ layout }),
});
const data = await res.json();
if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
const el = document.getElementById('debugResult');
const count = data.occupied.length;
const r = data.region;
let html = `<b>${layout}</b> ${data.cols}x${data.rows}`;
html += ` — ${count} occupied cell(s)`;
if (r) html += `<br><span style="color:#484f58">Region: (${r.x}, ${r.y}) ${r.width}x${r.height}</span>`;
if (count > 0) {
html += `<br>` + data.occupied.map(c => `(${c.row},${c.col})`).join(' ');
}
html += '<div class="grid-debug">';
if (data.image) {
html += `<img src="data:image/png;base64,${data.image}" alt="Grid capture" />`;
}
html += `<div class="grid-view" style="grid-template-columns:repeat(${data.cols},12px)">`;
const set = new Set(data.occupied.map(c => c.row + ',' + c.col));
for (let r = 0; r < data.rows; r++) {
for (let c = 0; c < data.cols; c++) {
html += `<div class="grid-cell${set.has(r+','+c) ? ' occupied' : ''}" title="(${r},${c})"></div>`;
}
}
html += '</div></div>';
el.innerHTML = html;
}
async function debugAngeOption(option) {
showDebugResult(`Clicking ANGE → ${option}...`);
const res = await fetch('/api/debug/click-then-click', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ first: 'ANGE', second: option }),
});
const data = await res.json();
if (!data.found && data.step === 'first') {
showDebugResult('ANGE not found on screen');
} else if (!data.found) {
showDebugResult(`"${option}" not found in ANGE menu (timed out)`);
} else {
showDebugResult(`Clicked "${option}" at (${data.position.x}, ${data.position.y})`);
}
}
async function debugHideout() {
showDebugResult('Sending /hideout...');
const res = await fetch('/api/debug/hideout', { method: 'POST' });
const data = await res.json();
showDebugResult(data.ok ? 'Sent /hideout command' : `Error: ${data.error}`);
}
async function debugScreenshot() {
const res = await fetch('/api/debug/screenshot', { method: 'POST' });
const data = await res.json();
@ -620,8 +734,8 @@
}
}
async function debugFindAndClick() {
const text = document.getElementById('debugTextInput').value.trim();
async function debugFindAndClick(directText) {
const text = directText || document.getElementById('debugTextInput').value.trim();
if (!text) return;
showDebugResult(`Finding and clicking "${text}"...`);
const res = await fetch('/api/debug/find-and-click', {

View file

@ -40,7 +40,7 @@ export class GameController {
// Clear any existing text
await this.inputSender.selectAll();
await sleep(50);
await this.inputSender.pressKey(VK.DELETE);
await this.inputSender.pressKey(VK.BACK);
await sleep(50);
// Type the message
@ -66,7 +66,7 @@ export class GameController {
// Clear any existing text
await this.inputSender.selectAll();
await sleep(50);
await this.inputSender.pressKey(VK.DELETE);
await this.inputSender.pressKey(VK.BACK);
await sleep(50);
// Paste
@ -80,7 +80,7 @@ export class GameController {
async goToHideout(): Promise<void> {
logger.info('Sending /hideout command');
await this.sendChat('/hideout');
await this.sendChatViaPaste('/hideout');
}
async ctrlRightClickAt(x: number, y: number): Promise<void> {

157
src/game/GridReader.ts Normal file
View file

@ -0,0 +1,157 @@
import { logger } from '../util/logger.js';
import type { OcrDaemon } from './OcrDaemon.js';
import type { Region } from '../types.js';
// ── Grid type definitions ───────────────────────────────────────────────────
export interface GridLayout {
region: Region;
cols: number;
rows: number;
}
export interface CellCoord {
row: number;
col: number;
x: number;
y: number;
}
export interface ScanResult {
layout: GridLayout;
occupied: CellCoord[];
}
// ── Calibrated grid layouts (2560×1440) ─────────────────────────────────────
export const GRID_LAYOUTS: Record<string, GridLayout> = {
/** Player inventory — always 12×5, right side (below equipment slots) */
inventory: {
region: { x: 1696, y: 788, width: 840, height: 350 },
cols: 12,
rows: 5,
},
/** Personal stash 12×12 — left side, tab not in folder */
stash12: {
region: { x: 23, y: 169, width: 840, height: 840 },
cols: 12,
rows: 12,
},
/** Personal stash 12×12 — left side, tab in folder */
stash12_folder: {
region: { x: 23, y: 216, width: 840, height: 840 },
cols: 12,
rows: 12,
},
/** Personal stash 24×24 (quad tab) — left side, tab not in folder */
stash24: {
region: { x: 23, y: 169, width: 840, height: 840 },
cols: 24,
rows: 24,
},
/** Personal stash 24×24 (quad tab) — left side, tab in folder */
stash24_folder: {
region: { x: 23, y: 216, width: 840, height: 840 },
cols: 24,
rows: 24,
},
/** Seller's public stash — always 12×12 */
seller: {
region: { x: 416, y: 299, width: 840, height: 840 },
cols: 12,
rows: 12,
},
/** NPC shop — 12×12 */
shop: {
region: { x: 23, y: 216, width: 840, height: 840 },
cols: 12,
rows: 12,
},
/** NPC vendor inventory — 12×12 */
vendor: {
region: { x: 416, y: 369, width: 840, height: 840 },
cols: 12,
rows: 12,
},
};
// Backward-compat exports
export const INVENTORY = GRID_LAYOUTS.inventory;
export const STASH_12x12 = GRID_LAYOUTS.stash12;
export const STASH_24x24 = GRID_LAYOUTS.stash24;
export const SELLER_12x12 = GRID_LAYOUTS.seller;
// ── GridReader ──────────────────────────────────────────────────────────────
export class GridReader {
constructor(private daemon: OcrDaemon) {}
/**
* Scan a named grid layout for occupied cells.
*/
async scan(layoutName: string, threshold?: number): Promise<ScanResult> {
const layout = GRID_LAYOUTS[layoutName];
if (!layout) throw new Error(`Unknown grid layout: ${layoutName}`);
const t = performance.now();
const occupied = await this.getOccupiedCells(layout, threshold);
const ms = (performance.now() - t).toFixed(0);
logger.info(
{ layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, ms },
'Grid scan complete',
);
return { layout, occupied };
}
/** Get the screen-space center of a grid cell */
getCellCenter(layout: GridLayout, row: number, col: number): { x: number; y: number } {
const cellW = layout.region.width / layout.cols;
const cellH = layout.region.height / layout.rows;
return {
x: Math.round(layout.region.x + col * cellW + cellW / 2),
y: Math.round(layout.region.y + row * cellH + cellH / 2),
};
}
/** Scan the grid and return which cells are occupied */
async getOccupiedCells(layout: GridLayout, threshold?: number): Promise<CellCoord[]> {
const t = performance.now();
const cells = await this.daemon.gridScan(
layout.region,
layout.cols,
layout.rows,
threshold,
);
const occupied: CellCoord[] = [];
for (let row = 0; row < cells.length; row++) {
for (let col = 0; col < cells[row].length; col++) {
if (cells[row][col]) {
const center = this.getCellCenter(layout, row, col);
occupied.push({ row, col, x: center.x, y: center.y });
}
}
}
const ms = (performance.now() - t).toFixed(0);
logger.info(
{ layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, ms },
'Grid scan complete',
);
return occupied;
}
/** Get all cell centers in the grid */
getAllCells(layout: GridLayout): CellCoord[] {
const cells: CellCoord[] = [];
for (let row = 0; row < layout.rows; row++) {
for (let col = 0; col < layout.cols; col++) {
const center = this.getCellCenter(layout, row, col);
cells.push({ row, col, x: center.x, y: center.y });
}
}
return cells;
}
}

View file

@ -202,7 +202,7 @@ export class InputSender {
y: start.y + dy * 0.75 + perpY * (Math.random() - 0.5) * spread,
};
const steps = clamp(Math.round(distance / 15), 15, 40);
const steps = clamp(Math.round(distance / 30), 8, 20);
for (let i = 1; i <= steps; i++) {
const rawT = i / steps;
@ -214,30 +214,30 @@ export class InputSender {
const jitterY = i < steps ? Math.round((Math.random() - 0.5) * 2) : 0;
this.moveMouseRaw(Math.round(pt.x) + jitterX, Math.round(pt.y) + jitterY);
await sleep(2 + Math.random() * 3); // 2-5ms between steps
await sleep(1 + Math.random() * 2); // 1-3ms between steps
}
// Final exact landing
this.moveMouseRaw(x, y);
await randomDelay(10, 25);
await randomDelay(5, 15);
}
async leftClick(x: number, y: number): Promise<void> {
await this.moveMouse(x, y);
await randomDelay(50, 100);
await randomDelay(20, 50);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_LEFTDOWN);
await randomDelay(30, 80);
await randomDelay(15, 40);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_LEFTUP);
await randomDelay(30, 60);
await randomDelay(15, 30);
}
async rightClick(x: number, y: number): Promise<void> {
await this.moveMouse(x, y);
await randomDelay(50, 100);
await randomDelay(20, 50);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_RIGHTDOWN);
await randomDelay(30, 80);
await randomDelay(15, 40);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_RIGHTUP);
await randomDelay(30, 60);
await randomDelay(15, 30);
}
async ctrlRightClick(x: number, y: number): Promise<void> {

View file

@ -24,10 +24,24 @@ export interface OcrResponse {
lines: OcrLine[];
}
export interface DetectGridResult {
detected: boolean;
region?: Region;
cols?: number;
rows?: number;
cellWidth?: number;
cellHeight?: number;
}
interface DaemonRequest {
cmd: string;
region?: Region;
path?: string;
cols?: number;
rows?: number;
threshold?: number;
minCellSize?: number;
maxCellSize?: number;
}
interface DaemonResponse {
@ -36,6 +50,13 @@ interface DaemonResponse {
text?: string;
lines?: OcrLine[];
image?: string;
cells?: boolean[][];
detected?: boolean;
region?: Region;
cols?: number;
rows?: number;
cellWidth?: number;
cellHeight?: number;
error?: string;
}
@ -85,6 +106,28 @@ export class OcrDaemon {
return Buffer.from(resp.image!, 'base64');
}
async gridScan(region: Region, cols: number, rows: number, threshold?: number): Promise<boolean[][]> {
const req: DaemonRequest = { cmd: 'grid', region, cols, rows };
if (threshold) req.threshold = threshold;
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
return resp.cells ?? [];
}
async detectGrid(region: Region, minCellSize?: number, maxCellSize?: number): Promise<DetectGridResult> {
const req: DaemonRequest = { cmd: 'detect-grid', region };
if (minCellSize) req.minCellSize = minCellSize;
if (maxCellSize) req.maxCellSize = maxCellSize;
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
return {
detected: resp.detected ?? false,
region: resp.region,
cols: resp.cols,
rows: resp.rows,
cellWidth: resp.cellWidth,
cellHeight: resp.cellHeight,
};
}
async saveScreenshot(path: string, region?: Region): Promise<void> {
const req: DaemonRequest = { cmd: 'screenshot', path };
if (region) req.region = region;

View file

@ -2,6 +2,7 @@ import { mkdir } from 'fs/promises';
import { join } from 'path';
import { logger } from '../util/logger.js';
import { OcrDaemon, type OcrResponse } from './OcrDaemon.js';
import { GridReader, type GridLayout, type CellCoord } from './GridReader.js';
import type { Region } from '../types.js';
function elapsed(start: number): string {
@ -10,6 +11,7 @@ function elapsed(start: number): string {
export class ScreenReader {
private daemon = new OcrDaemon();
readonly grid = new GridReader(this.daemon);
// ── Screenshot capture ──────────────────────────────────────────────

View file

@ -63,6 +63,14 @@ program
const gameController = new GameController(config);
dashboard.setDebugDeps({ screenReader, gameController });
// Go to hideout on startup
dashboard.broadcastLog('info', 'Sending /hideout command...');
await gameController.focusGame();
await gameController.goToHideout();
bot.state = 'IN_HIDEOUT';
dashboard.broadcastStatus();
dashboard.broadcastLog('info', 'In hideout, ready to trade');
const logWatcher = new ClientLogWatcher(config.poe2LogPath);
await logWatcher.start();
dashboard.broadcastLog('info', 'Watching Client.txt for game events');