@@ -588,6 +645,63 @@
}
// Debug functions
+ async function debugGridScan(layout) {
+ showDebugResult(`Scanning ${layout}...`);
+ const res = await fetch('/api/debug/grid-scan', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ layout }),
+ });
+ const data = await res.json();
+ if (!data.ok) { showDebugResult(`Error: ${data.error}`); return; }
+ const el = document.getElementById('debugResult');
+ const count = data.occupied.length;
+ const r = data.region;
+ let html = `${layout} ${data.cols}x${data.rows}`;
+ html += ` — ${count} occupied cell(s)`;
+ if (r) html += ` Region: (${r.x}, ${r.y}) ${r.width}x${r.height}`;
+ if (count > 0) {
+ html += ` ` + data.occupied.map(c => `(${c.row},${c.col})`).join(' ');
+ }
+ html += '
';
+ if (data.image) {
+ html += ``;
+ }
+ html += `
`;
+ const set = new Set(data.occupied.map(c => c.row + ',' + c.col));
+ for (let r = 0; r < data.rows; r++) {
+ for (let c = 0; c < data.cols; c++) {
+ html += ``;
+ }
+ }
+ html += '
';
+ el.innerHTML = html;
+ }
+
+ async function debugAngeOption(option) {
+ showDebugResult(`Clicking ANGE → ${option}...`);
+ const res = await fetch('/api/debug/click-then-click', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ first: 'ANGE', second: option }),
+ });
+ const data = await res.json();
+ if (!data.found && data.step === 'first') {
+ showDebugResult('ANGE not found on screen');
+ } else if (!data.found) {
+ showDebugResult(`"${option}" not found in ANGE menu (timed out)`);
+ } else {
+ showDebugResult(`Clicked "${option}" at (${data.position.x}, ${data.position.y})`);
+ }
+ }
+
+ async function debugHideout() {
+ showDebugResult('Sending /hideout...');
+ const res = await fetch('/api/debug/hideout', { method: 'POST' });
+ const data = await res.json();
+ showDebugResult(data.ok ? 'Sent /hideout command' : `Error: ${data.error}`);
+ }
+
async function debugScreenshot() {
const res = await fetch('/api/debug/screenshot', { method: 'POST' });
const data = await res.json();
@@ -620,8 +734,8 @@
}
}
- async function debugFindAndClick() {
- const text = document.getElementById('debugTextInput').value.trim();
+ async function debugFindAndClick(directText) {
+ const text = directText || document.getElementById('debugTextInput').value.trim();
if (!text) return;
showDebugResult(`Finding and clicking "${text}"...`);
const res = await fetch('/api/debug/find-and-click', {
diff --git a/src/game/GameController.ts b/src/game/GameController.ts
index dec4463..ddb7fa4 100644
--- a/src/game/GameController.ts
+++ b/src/game/GameController.ts
@@ -40,7 +40,7 @@ export class GameController {
// Clear any existing text
await this.inputSender.selectAll();
await sleep(50);
- await this.inputSender.pressKey(VK.DELETE);
+ await this.inputSender.pressKey(VK.BACK);
await sleep(50);
// Type the message
@@ -66,7 +66,7 @@ export class GameController {
// Clear any existing text
await this.inputSender.selectAll();
await sleep(50);
- await this.inputSender.pressKey(VK.DELETE);
+ await this.inputSender.pressKey(VK.BACK);
await sleep(50);
// Paste
@@ -80,7 +80,7 @@ export class GameController {
async goToHideout(): Promise {
logger.info('Sending /hideout command');
- await this.sendChat('/hideout');
+ await this.sendChatViaPaste('/hideout');
}
async ctrlRightClickAt(x: number, y: number): Promise {
diff --git a/src/game/GridReader.ts b/src/game/GridReader.ts
new file mode 100644
index 0000000..35fd4bc
--- /dev/null
+++ b/src/game/GridReader.ts
@@ -0,0 +1,157 @@
+import { logger } from '../util/logger.js';
+import type { OcrDaemon } from './OcrDaemon.js';
+import type { Region } from '../types.js';
+
+// ── Grid type definitions ───────────────────────────────────────────────────
+
+export interface GridLayout {
+ region: Region;
+ cols: number;
+ rows: number;
+}
+
+export interface CellCoord {
+ row: number;
+ col: number;
+ x: number;
+ y: number;
+}
+
+export interface ScanResult {
+ layout: GridLayout;
+ occupied: CellCoord[];
+}
+
+// ── Calibrated grid layouts (2560×1440) ─────────────────────────────────────
+
+export const GRID_LAYOUTS: Record = {
+ /** Player inventory — always 12×5, right side (below equipment slots) */
+ inventory: {
+ region: { x: 1696, y: 788, width: 840, height: 350 },
+ cols: 12,
+ rows: 5,
+ },
+ /** Personal stash 12×12 — left side, tab not in folder */
+ stash12: {
+ region: { x: 23, y: 169, width: 840, height: 840 },
+ cols: 12,
+ rows: 12,
+ },
+ /** Personal stash 12×12 — left side, tab in folder */
+ stash12_folder: {
+ region: { x: 23, y: 216, width: 840, height: 840 },
+ cols: 12,
+ rows: 12,
+ },
+ /** Personal stash 24×24 (quad tab) — left side, tab not in folder */
+ stash24: {
+ region: { x: 23, y: 169, width: 840, height: 840 },
+ cols: 24,
+ rows: 24,
+ },
+ /** Personal stash 24×24 (quad tab) — left side, tab in folder */
+ stash24_folder: {
+ region: { x: 23, y: 216, width: 840, height: 840 },
+ cols: 24,
+ rows: 24,
+ },
+ /** Seller's public stash — always 12×12 */
+ seller: {
+ region: { x: 416, y: 299, width: 840, height: 840 },
+ cols: 12,
+ rows: 12,
+ },
+ /** NPC shop — 12×12 */
+ shop: {
+ region: { x: 23, y: 216, width: 840, height: 840 },
+ cols: 12,
+ rows: 12,
+ },
+ /** NPC vendor inventory — 12×12 */
+ vendor: {
+ region: { x: 416, y: 369, width: 840, height: 840 },
+ cols: 12,
+ rows: 12,
+ },
+};
+
+// Backward-compat exports
+export const INVENTORY = GRID_LAYOUTS.inventory;
+export const STASH_12x12 = GRID_LAYOUTS.stash12;
+export const STASH_24x24 = GRID_LAYOUTS.stash24;
+export const SELLER_12x12 = GRID_LAYOUTS.seller;
+
+// ── GridReader ──────────────────────────────────────────────────────────────
+
+export class GridReader {
+ constructor(private daemon: OcrDaemon) {}
+
+ /**
+ * Scan a named grid layout for occupied cells.
+ */
+ async scan(layoutName: string, threshold?: number): Promise {
+ const layout = GRID_LAYOUTS[layoutName];
+ if (!layout) throw new Error(`Unknown grid layout: ${layoutName}`);
+
+ const t = performance.now();
+ const occupied = await this.getOccupiedCells(layout, threshold);
+
+ const ms = (performance.now() - t).toFixed(0);
+ logger.info(
+ { layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, ms },
+ 'Grid scan complete',
+ );
+
+ return { layout, occupied };
+ }
+
+ /** Get the screen-space center of a grid cell */
+ getCellCenter(layout: GridLayout, row: number, col: number): { x: number; y: number } {
+ const cellW = layout.region.width / layout.cols;
+ const cellH = layout.region.height / layout.rows;
+ return {
+ x: Math.round(layout.region.x + col * cellW + cellW / 2),
+ y: Math.round(layout.region.y + row * cellH + cellH / 2),
+ };
+ }
+
+ /** Scan the grid and return which cells are occupied */
+ async getOccupiedCells(layout: GridLayout, threshold?: number): Promise {
+ const t = performance.now();
+ const cells = await this.daemon.gridScan(
+ layout.region,
+ layout.cols,
+ layout.rows,
+ threshold,
+ );
+
+ const occupied: CellCoord[] = [];
+ for (let row = 0; row < cells.length; row++) {
+ for (let col = 0; col < cells[row].length; col++) {
+ if (cells[row][col]) {
+ const center = this.getCellCenter(layout, row, col);
+ occupied.push({ row, col, x: center.x, y: center.y });
+ }
+ }
+ }
+
+ const ms = (performance.now() - t).toFixed(0);
+ logger.info(
+ { layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, ms },
+ 'Grid scan complete',
+ );
+ return occupied;
+ }
+
+ /** Get all cell centers in the grid */
+ getAllCells(layout: GridLayout): CellCoord[] {
+ const cells: CellCoord[] = [];
+ for (let row = 0; row < layout.rows; row++) {
+ for (let col = 0; col < layout.cols; col++) {
+ const center = this.getCellCenter(layout, row, col);
+ cells.push({ row, col, x: center.x, y: center.y });
+ }
+ }
+ return cells;
+ }
+}
diff --git a/src/game/InputSender.ts b/src/game/InputSender.ts
index 585e18d..410ef26 100644
--- a/src/game/InputSender.ts
+++ b/src/game/InputSender.ts
@@ -202,7 +202,7 @@ export class InputSender {
y: start.y + dy * 0.75 + perpY * (Math.random() - 0.5) * spread,
};
- const steps = clamp(Math.round(distance / 15), 15, 40);
+ const steps = clamp(Math.round(distance / 30), 8, 20);
for (let i = 1; i <= steps; i++) {
const rawT = i / steps;
@@ -214,30 +214,30 @@ export class InputSender {
const jitterY = i < steps ? Math.round((Math.random() - 0.5) * 2) : 0;
this.moveMouseRaw(Math.round(pt.x) + jitterX, Math.round(pt.y) + jitterY);
- await sleep(2 + Math.random() * 3); // 2-5ms between steps
+ await sleep(1 + Math.random() * 2); // 1-3ms between steps
}
// Final exact landing
this.moveMouseRaw(x, y);
- await randomDelay(10, 25);
+ await randomDelay(5, 15);
}
async leftClick(x: number, y: number): Promise {
await this.moveMouse(x, y);
- await randomDelay(50, 100);
+ await randomDelay(20, 50);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_LEFTDOWN);
- await randomDelay(30, 80);
+ await randomDelay(15, 40);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_LEFTUP);
- await randomDelay(30, 60);
+ await randomDelay(15, 30);
}
async rightClick(x: number, y: number): Promise {
await this.moveMouse(x, y);
- await randomDelay(50, 100);
+ await randomDelay(20, 50);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_RIGHTDOWN);
- await randomDelay(30, 80);
+ await randomDelay(15, 40);
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_RIGHTUP);
- await randomDelay(30, 60);
+ await randomDelay(15, 30);
}
async ctrlRightClick(x: number, y: number): Promise {
diff --git a/src/game/OcrDaemon.ts b/src/game/OcrDaemon.ts
index 256e7b5..5d1d162 100644
--- a/src/game/OcrDaemon.ts
+++ b/src/game/OcrDaemon.ts
@@ -24,10 +24,24 @@ export interface OcrResponse {
lines: OcrLine[];
}
+export interface DetectGridResult {
+ detected: boolean;
+ region?: Region;
+ cols?: number;
+ rows?: number;
+ cellWidth?: number;
+ cellHeight?: number;
+}
+
interface DaemonRequest {
cmd: string;
region?: Region;
path?: string;
+ cols?: number;
+ rows?: number;
+ threshold?: number;
+ minCellSize?: number;
+ maxCellSize?: number;
}
interface DaemonResponse {
@@ -36,6 +50,13 @@ interface DaemonResponse {
text?: string;
lines?: OcrLine[];
image?: string;
+ cells?: boolean[][];
+ detected?: boolean;
+ region?: Region;
+ cols?: number;
+ rows?: number;
+ cellWidth?: number;
+ cellHeight?: number;
error?: string;
}
@@ -85,6 +106,28 @@ export class OcrDaemon {
return Buffer.from(resp.image!, 'base64');
}
+ async gridScan(region: Region, cols: number, rows: number, threshold?: number): Promise {
+ const req: DaemonRequest = { cmd: 'grid', region, cols, rows };
+ if (threshold) req.threshold = threshold;
+ const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
+ return resp.cells ?? [];
+ }
+
+ async detectGrid(region: Region, minCellSize?: number, maxCellSize?: number): Promise {
+ const req: DaemonRequest = { cmd: 'detect-grid', region };
+ if (minCellSize) req.minCellSize = minCellSize;
+ if (maxCellSize) req.maxCellSize = maxCellSize;
+ const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
+ return {
+ detected: resp.detected ?? false,
+ region: resp.region,
+ cols: resp.cols,
+ rows: resp.rows,
+ cellWidth: resp.cellWidth,
+ cellHeight: resp.cellHeight,
+ };
+ }
+
async saveScreenshot(path: string, region?: Region): Promise {
const req: DaemonRequest = { cmd: 'screenshot', path };
if (region) req.region = region;
diff --git a/src/game/ScreenReader.ts b/src/game/ScreenReader.ts
index bc34672..be451e4 100644
--- a/src/game/ScreenReader.ts
+++ b/src/game/ScreenReader.ts
@@ -2,6 +2,7 @@ import { mkdir } from 'fs/promises';
import { join } from 'path';
import { logger } from '../util/logger.js';
import { OcrDaemon, type OcrResponse } from './OcrDaemon.js';
+import { GridReader, type GridLayout, type CellCoord } from './GridReader.js';
import type { Region } from '../types.js';
function elapsed(start: number): string {
@@ -10,6 +11,7 @@ function elapsed(start: number): string {
export class ScreenReader {
private daemon = new OcrDaemon();
+ readonly grid = new GridReader(this.daemon);
// ── Screenshot capture ──────────────────────────────────────────────
diff --git a/src/index.ts b/src/index.ts
index 01990d2..9c86d3b 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -63,6 +63,14 @@ program
const gameController = new GameController(config);
dashboard.setDebugDeps({ screenReader, gameController });
+ // Go to hideout on startup
+ dashboard.broadcastLog('info', 'Sending /hideout command...');
+ await gameController.focusGame();
+ await gameController.goToHideout();
+ bot.state = 'IN_HIDEOUT';
+ dashboard.broadcastStatus();
+ dashboard.broadcastLog('info', 'In hideout, ready to trade');
+
const logWatcher = new ClientLogWatcher(config.poe2LogPath);
await logWatcher.start();
dashboard.broadcastLog('info', 'Watching Client.txt for game events');
diff --git a/tools/OcrDaemon/Program.cs b/tools/OcrDaemon/Program.cs
index ea38c16..d74c87a 100644
--- a/tools/OcrDaemon/Program.cs
+++ b/tools/OcrDaemon/Program.cs
@@ -57,6 +57,12 @@ while ((line = stdin.ReadLine()) != null)
case "capture":
HandleCapture(request);
break;
+ case "grid":
+ HandleGrid(request);
+ break;
+ case "detect-grid":
+ HandleDetectGrid(request);
+ break;
default:
WriteResponse(new ErrorResponse($"Unknown command: {request.Cmd}"));
break;
@@ -74,7 +80,7 @@ return 0;
void HandleOcr(Request req, OcrEngine engine)
{
- using var bitmap = CaptureScreen(req.Region);
+ using var bitmap = CaptureOrLoad(req.File, req.Region);
var softwareBitmap = BitmapToSoftwareBitmap(bitmap);
var result = engine.RecognizeAsync(softwareBitmap).AsTask().GetAwaiter().GetResult();
@@ -107,7 +113,7 @@ void HandleScreenshot(Request req)
return;
}
- using var bitmap = CaptureScreen(req.Region);
+ using var bitmap = CaptureOrLoad(req.File, req.Region);
var format = GetImageFormat(req.Path);
bitmap.Save(req.Path, format);
WriteResponse(new OkResponse());
@@ -115,15 +121,573 @@ void HandleScreenshot(Request req)
void HandleCapture(Request req)
{
- using var bitmap = CaptureScreen(req.Region);
+ using var bitmap = CaptureOrLoad(req.File, req.Region);
using var ms = new MemoryStream();
bitmap.Save(ms, ImageFormat.Png);
var base64 = Convert.ToBase64String(ms.ToArray());
WriteResponse(new CaptureResponse { Image = base64 });
}
+// Pre-loaded empty cell templates (loaded lazily on first grid scan)
+byte[]? emptyTemplate70Gray = null;
+int emptyTemplate70W = 0, emptyTemplate70H = 0;
+byte[]? emptyTemplate35Gray = null;
+int emptyTemplate35W = 0, emptyTemplate35H = 0;
+
+void LoadTemplatesIfNeeded()
+{
+ if (emptyTemplate70Gray != null) return;
+
+ // Look for templates relative to exe directory
+ var exeDir = AppContext.BaseDirectory;
+ // Templates are in assets/ at project root — walk up from bin/Release/net8.0-.../
+ var projectRoot = System.IO.Path.GetFullPath(System.IO.Path.Combine(exeDir, "..", "..", "..", "..", ".."));
+ var t70Path = System.IO.Path.Combine(projectRoot, "assets", "empty70.png");
+ var t35Path = System.IO.Path.Combine(projectRoot, "assets", "empty35.png");
+
+ if (System.IO.File.Exists(t70Path))
+ {
+ using var bmp = new Bitmap(t70Path);
+ emptyTemplate70W = bmp.Width;
+ emptyTemplate70H = bmp.Height;
+ emptyTemplate70Gray = BitmapToGray(bmp);
+ }
+ if (System.IO.File.Exists(t35Path))
+ {
+ using var bmp = new Bitmap(t35Path);
+ emptyTemplate35W = bmp.Width;
+ emptyTemplate35H = bmp.Height;
+ emptyTemplate35Gray = BitmapToGray(bmp);
+ }
+}
+
+byte[] BitmapToGray(Bitmap bmp)
+{
+ int w = bmp.Width, h = bmp.Height;
+ var data = bmp.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
+ byte[] pixels = new byte[data.Stride * h];
+ Marshal.Copy(data.Scan0, pixels, 0, pixels.Length);
+ bmp.UnlockBits(data);
+ int stride = data.Stride;
+
+ byte[] gray = new byte[w * h];
+ for (int y = 0; y < h; y++)
+ for (int x = 0; x < w; x++)
+ {
+ int i = y * stride + x * 4;
+ gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
+ }
+ return gray;
+}
+
+void HandleGrid(Request req)
+{
+ if (req.Region == null || req.Cols <= 0 || req.Rows <= 0)
+ {
+ WriteResponse(new ErrorResponse("grid command requires region, cols, rows"));
+ return;
+ }
+
+ LoadTemplatesIfNeeded();
+
+ using var bitmap = CaptureOrLoad(req.File, req.Region);
+ int cols = req.Cols;
+ int rows = req.Rows;
+ float cellW = (float)bitmap.Width / cols;
+ float cellH = (float)bitmap.Height / rows;
+
+ // Pick the right empty template based on cell size
+ int nominalCell = (int)Math.Round(cellW);
+ byte[]? templateGray;
+ int templateW, templateH;
+ if (nominalCell <= 40 && emptyTemplate35Gray != null)
+ {
+ templateGray = emptyTemplate35Gray;
+ templateW = emptyTemplate35W;
+ templateH = emptyTemplate35H;
+ }
+ else if (emptyTemplate70Gray != null)
+ {
+ templateGray = emptyTemplate70Gray;
+ templateW = emptyTemplate70W;
+ templateH = emptyTemplate70H;
+ }
+ else
+ {
+ WriteResponse(new ErrorResponse("Empty cell templates not found in assets/"));
+ return;
+ }
+
+ // Convert captured bitmap to grayscale
+ byte[] captureGray = BitmapToGray(bitmap);
+ int captureW = bitmap.Width;
+
+ // Border to skip (outer pixels may differ between cells)
+ int border = Math.Max(2, nominalCell / 10);
+
+ // Pre-compute template average for the inner region
+ long templateSum = 0;
+ int innerCount = 0;
+ for (int ty = border; ty < templateH - border; ty++)
+ for (int tx = border; tx < templateW - border; tx++)
+ {
+ templateSum += templateGray[ty * templateW + tx];
+ innerCount++;
+ }
+
+ // Threshold for mean absolute difference — default 6
+ double diffThreshold = req.Threshold > 0 ? req.Threshold : 2;
+ bool debug = req.Debug;
+
+ if (debug) Console.Error.WriteLine($"Grid: {cols}x{rows}, cellW={cellW:F1}, cellH={cellH:F1}, border={border}, threshold={diffThreshold}");
+
+ var cells = new List>();
+ for (int row = 0; row < rows; row++)
+ {
+ var rowList = new List();
+ var debugDiffs = new List();
+ for (int col = 0; col < cols; col++)
+ {
+ int cx0 = (int)(col * cellW);
+ int cy0 = (int)(row * cellH);
+ int cw = (int)Math.Min(cellW, captureW - cx0);
+ int ch = (int)Math.Min(cellH, bitmap.Height - cy0);
+
+ // Compare inner pixels of cell vs template
+ long diffSum = 0;
+ int compared = 0;
+ int innerW = Math.Min(cw, templateW) - border;
+ int innerH = Math.Min(ch, templateH) - border;
+ for (int py = border; py < innerH; py++)
+ {
+ for (int px = border; px < innerW; px++)
+ {
+ int cellVal = captureGray[(cy0 + py) * captureW + (cx0 + px)];
+ int tmplVal = templateGray[py * templateW + px];
+ diffSum += Math.Abs(cellVal - tmplVal);
+ compared++;
+ }
+ }
+ double meanDiff = compared > 0 ? (double)diffSum / compared : 0;
+ bool occupied = meanDiff > diffThreshold;
+ rowList.Add(occupied);
+ if (debug) debugDiffs.Add($"{meanDiff,5:F1}{(occupied ? "*" : " ")}");
+ }
+ cells.Add(rowList);
+ if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}");
+ }
+
+ WriteResponse(new GridResponse { Cells = cells });
+}
+
+void HandleDetectGrid(Request req)
+{
+ if (req.Region == null)
+ {
+ WriteResponse(new ErrorResponse("detect-grid requires region"));
+ return;
+ }
+
+ int minCell = req.MinCellSize > 0 ? req.MinCellSize : 20;
+ int maxCell = req.MaxCellSize > 0 ? req.MaxCellSize : 70;
+ bool debug = req.Debug;
+
+ Bitmap bitmap = CaptureOrLoad(req.File, req.Region);
+ int w = bitmap.Width;
+ int h = bitmap.Height;
+
+ var bmpData = bitmap.LockBits(
+ new Rectangle(0, 0, w, h),
+ ImageLockMode.ReadOnly,
+ PixelFormat.Format32bppArgb
+ );
+ byte[] pixels = new byte[bmpData.Stride * h];
+ Marshal.Copy(bmpData.Scan0, pixels, 0, pixels.Length);
+ bitmap.UnlockBits(bmpData);
+ int stride = bmpData.Stride;
+
+ byte[] gray = new byte[w * h];
+ for (int y = 0; y < h; y++)
+ for (int x = 0; x < w; x++)
+ {
+ int i = y * stride + x * 4;
+ gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
+ }
+
+ bitmap.Dispose();
+
+ // ── Pass 1: Scan horizontal bands using "very dark pixel density" ──
+ // Grid lines are nearly all very dark (density ~0.9), cell interiors are
+ // partially dark (0.3-0.5), game world is mostly bright (density ~0.05).
+ // This creates clear periodic peaks at grid line positions.
+ int bandH = 200;
+ int bandStep = 40;
+ const int veryDarkPixelThresh = 12; // pixels below this brightness = "very dark"
+ const double gridSegThresh = 0.25; // density above this = potential grid column
+
+ var candidates = new List<(int bandY, int cellW, double hAc, int hLeft, int hRight)>();
+
+ for (int by = 0; by + bandH <= h; by += bandStep)
+ {
+ // "Very dark pixel density" per column: fraction of pixels below threshold
+ double[] darkDensity = new double[w];
+ for (int x = 0; x < w; x++)
+ {
+ int count = 0;
+ for (int y = by; y < by + bandH; y++)
+ {
+ if (gray[y * w + x] < veryDarkPixelThresh) count++;
+ }
+ darkDensity[x] = (double)count / bandH;
+ }
+
+ // Find segments where density > gridSegThresh (grid panel regions)
+ var gridSegs = FindDarkDensitySegments(darkDensity, gridSegThresh, 200);
+
+ foreach (var (segLeft, segRight) in gridSegs)
+ {
+ // Extract segment and run AC
+ int segLen = segRight - segLeft;
+ double[] segment = new double[segLen];
+ Array.Copy(darkDensity, segLeft, segment, 0, segLen);
+
+ var (period, acScore) = FindPeriodWithScore(segment, minCell, maxCell);
+
+ if (period <= 0) continue;
+
+ // FindGridExtent within the segment
+ var (extLeft, extRight) = FindGridExtent(segment, period);
+ if (extLeft < 0) continue;
+
+ // Map back to full image coordinates
+ int absLeft = segLeft + extLeft;
+ int absRight = segLeft + extRight;
+ int extent = absRight - absLeft;
+
+ // Require at least 8 cells wide AND 200px absolute minimum
+ if (extent < period * 8 || extent < 200) continue;
+
+ if (debug) Console.Error.WriteLine(
+ $" Band y={by}: seg=[{segLeft}-{segRight}] period={period}, AC={acScore:F3}, " +
+ $"extent={absLeft}-{absRight}={extent}px ({extent / period} cells)");
+
+ candidates.Add((by, period, acScore, absLeft, absRight));
+ }
+ }
+
+ if (debug) Console.Error.WriteLine($"Pass 1: {candidates.Count} candidates");
+
+ // Sort by score = AC * extent (prefer large strongly-periodic areas)
+ candidates.Sort((a, b) =>
+ {
+ double sa = a.hAc * (a.hRight - a.hLeft);
+ double sb = b.hAc * (b.hRight - b.hLeft);
+ return sb.CompareTo(sa);
+ });
+
+ // ── Pass 2: Verify vertical periodicity ──
+ foreach (var cand in candidates.Take(10))
+ {
+ int colSpan = cand.hRight - cand.hLeft;
+ if (colSpan < cand.cellW * 3) continue;
+
+ // Row "very dark pixel density" within the detected column range
+ double[] rowDensity = new double[h];
+ for (int y = 0; y < h; y++)
+ {
+ int count = 0;
+ for (int x = cand.hLeft; x < cand.hRight; x++)
+ {
+ if (gray[y * w + x] < veryDarkPixelThresh) count++;
+ }
+ rowDensity[y] = (double)count / colSpan;
+ }
+
+ // Find grid panel vertical segment
+ var vGridSegs = FindDarkDensitySegments(rowDensity, gridSegThresh, 100);
+ if (vGridSegs.Count == 0) continue;
+
+ // Use the largest segment
+ var (vSegTop, vSegBottom) = vGridSegs.OrderByDescending(s => s.end - s.start).First();
+ int vSegLen = vSegBottom - vSegTop;
+ double[] vSegment = new double[vSegLen];
+ Array.Copy(rowDensity, vSegTop, vSegment, 0, vSegLen);
+
+ var (cellH, vAc) = FindPeriodWithScore(vSegment, minCell, maxCell);
+ if (cellH <= 0) continue;
+
+ var (extTop, extBottom) = FindGridExtent(vSegment, cellH);
+ if (extTop < 0) continue;
+
+ int top = vSegTop + extTop;
+ int bottom = vSegTop + extBottom;
+ int vExtent = bottom - top;
+
+ // Require at least 3 rows tall AND 100px absolute minimum
+ if (vExtent < cellH * 3 || vExtent < 100) continue;
+
+ if (debug) Console.Error.WriteLine(
+ $" 2D candidate: cellW={cand.cellW}, cellH={cellH}, " +
+ $"region=({cand.hLeft},{top})-({cand.hRight},{bottom}), " +
+ $"vAC={vAc:F3}, extent={vExtent}px ({vExtent / cellH} rows)");
+
+ // ── Found a valid 2D grid ──
+ int gridW = cand.hRight - cand.hLeft;
+ int gridH = bottom - top;
+ int cols = Math.Max(2, (int)Math.Round((double)gridW / cand.cellW));
+ int rows = Math.Max(2, (int)Math.Round((double)gridH / cellH));
+
+ // Snap grid dimensions to exact multiples of cell size
+ gridW = cols * cand.cellW;
+ gridH = rows * cellH;
+
+ if (debug) Console.Error.WriteLine(
+ $" => cols={cols}, rows={rows}, gridW={gridW}, gridH={gridH}");
+
+ WriteResponse(new DetectGridResponse
+ {
+ Detected = true,
+ Region = new RegionRect
+ {
+ X = req.Region.X + cand.hLeft,
+ Y = req.Region.Y + top,
+ Width = gridW,
+ Height = gridH,
+ },
+ Cols = cols,
+ Rows = rows,
+ CellWidth = Math.Round((double)gridW / cols, 1),
+ CellHeight = Math.Round((double)gridH / rows, 1),
+ });
+ return;
+ }
+
+ if (debug) Console.Error.WriteLine(" No valid 2D grid found");
+ WriteResponse(new DetectGridResponse { Detected = false });
+}
+
+/// Find the dominant period in a signal using autocorrelation.
+/// Returns (period, score) where score is the autocorrelation strength.
+(int period, double score) FindPeriodWithScore(double[] signal, int minPeriod, int maxPeriod)
+{
+ int n = signal.Length;
+ if (n < minPeriod * 3) return (-1, 0);
+
+ double mean = signal.Average();
+ double variance = 0;
+ for (int i = 0; i < n; i++)
+ variance += (signal[i] - mean) * (signal[i] - mean);
+ if (variance < 1.0) return (-1, 0);
+
+ int maxLag = Math.Min(maxPeriod, n / 3);
+ double[] ac = new double[maxLag + 1];
+ for (int lag = minPeriod; lag <= maxLag; lag++)
+ {
+ double sum = 0;
+ for (int i = 0; i < n - lag; i++)
+ sum += (signal[i] - mean) * (signal[i + lag] - mean);
+ ac[lag] = sum / variance;
+ }
+
+ // Find the first significant peak — this is the fundamental period.
+ // Using "first" avoids picking harmonics (2x, 3x) or unrelated larger patterns.
+ for (int lag = minPeriod + 1; lag < maxLag; lag++)
+ {
+ if (ac[lag] > 0.01 && ac[lag] >= ac[lag - 1] && ac[lag] >= ac[lag + 1])
+ return (lag, ac[lag]);
+ }
+
+ return (-1, 0);
+}
+
+/// Find contiguous segments where values are ABOVE threshold.
+/// Used to find grid panel regions by density of very dark pixels.
+/// Allows brief gaps (up to 5px) to handle grid borders.
+List<(int start, int end)> FindDarkDensitySegments(double[] profile, double threshold, int minLength)
+{
+ var segments = new List<(int start, int end)>();
+ int n = profile.Length;
+ int curStart = -1;
+ int maxGap = 5;
+ int gapCount = 0;
+
+ for (int i = 0; i < n; i++)
+ {
+ if (profile[i] >= threshold)
+ {
+ if (curStart < 0) curStart = i;
+ gapCount = 0;
+ }
+ else
+ {
+ if (curStart >= 0)
+ {
+ gapCount++;
+ if (gapCount > maxGap)
+ {
+ int end = i - gapCount;
+ if (end - curStart >= minLength)
+ segments.Add((curStart, end));
+ curStart = -1;
+ gapCount = 0;
+ }
+ }
+ }
+ }
+ if (curStart >= 0)
+ {
+ int end = gapCount > 0 ? n - gapCount : n;
+ if (end - curStart >= minLength)
+ segments.Add((curStart, end));
+ }
+
+ return segments;
+}
+
+/// Debug: find the top N AC peaks in a signal
+List<(int lag, double ac)> FindTopAcPeaks(double[] signal, int minPeriod, int maxPeriod, int topN)
+{
+ int n = signal.Length;
+ if (n < minPeriod * 3) return [];
+
+ double mean = signal.Average();
+ double variance = 0;
+ for (int i = 0; i < n; i++)
+ variance += (signal[i] - mean) * (signal[i] - mean);
+ if (variance < 1.0) return [];
+
+ int maxLag = Math.Min(maxPeriod, n / 3);
+ var peaks = new List<(int lag, double ac)>();
+ double[] ac = new double[maxLag + 1];
+ for (int lag = minPeriod; lag <= maxLag; lag++)
+ {
+ double sum = 0;
+ for (int i = 0; i < n - lag; i++)
+ sum += (signal[i] - mean) * (signal[i + lag] - mean);
+ ac[lag] = sum / variance;
+ }
+ for (int lag = minPeriod + 1; lag < maxLag; lag++)
+ {
+ if (ac[lag] >= ac[lag - 1] && ac[lag] >= ac[lag + 1] && ac[lag] > 0.005)
+ peaks.Add((lag, ac[lag]));
+ }
+ peaks.Sort((a, b) => b.ac.CompareTo(a.ac));
+ return peaks.Take(topN).ToList();
+}
+
+/// Find the extent of the grid in a 1D profile using local autocorrelation
+/// at the specific detected period. Only regions where the signal actually
+/// repeats at the given period will score high — much more precise than variance.
+(int start, int end) FindGridExtent(double[] signal, int period)
+{
+ int n = signal.Length;
+ int halfWin = period * 2; // window radius: 2 periods each side
+ if (n < halfWin * 2 + period) return (-1, -1);
+
+ // Compute local AC at the specific lag=period in a sliding window
+ double[] localAc = new double[n];
+ for (int center = halfWin; center < n - halfWin; center++)
+ {
+ int wStart = center - halfWin;
+ int wEnd = center + halfWin;
+ int count = wEnd - wStart;
+
+ // Local mean
+ double sum = 0;
+ for (int i = wStart; i < wEnd; i++)
+ sum += signal[i];
+ double mean = sum / count;
+
+ // Local variance
+ double varSum = 0;
+ for (int i = wStart; i < wEnd; i++)
+ varSum += (signal[i] - mean) * (signal[i] - mean);
+
+ if (varSum < 1.0) continue;
+
+ // AC at the specific lag=period
+ double acSum = 0;
+ for (int i = wStart; i < wEnd - period; i++)
+ acSum += (signal[i] - mean) * (signal[i + period] - mean);
+
+ localAc[center] = Math.Max(0, acSum / varSum);
+ }
+
+ // Find the longest contiguous run above threshold
+ double maxAc = 0;
+ for (int i = 0; i < n; i++)
+ if (localAc[i] > maxAc) maxAc = localAc[i];
+ if (maxAc < 0.02) return (-1, -1);
+
+ double threshold = maxAc * 0.25;
+
+ int bestStart = -1, bestEnd = -1, bestLen = 0;
+ int curStart = -1;
+ for (int i = 0; i < n; i++)
+ {
+ if (localAc[i] > threshold)
+ {
+ if (curStart < 0) curStart = i;
+ }
+ else
+ {
+ if (curStart >= 0)
+ {
+ int len = i - curStart;
+ if (len > bestLen)
+ {
+ bestLen = len;
+ bestStart = curStart;
+ bestEnd = i;
+ }
+ curStart = -1;
+ }
+ }
+ }
+ // Handle run extending to end of signal
+ if (curStart >= 0)
+ {
+ int len = n - curStart;
+ if (len > bestLen)
+ {
+ bestStart = curStart;
+ bestEnd = n;
+ }
+ }
+
+ if (bestStart < 0) return (-1, -1);
+
+ // Small extension to include cell borders at edges
+ bestStart = Math.Max(0, bestStart - period / 4);
+ bestEnd = Math.Min(n - 1, bestEnd + period / 4);
+
+ return (bestStart, bestEnd);
+}
+
// ── Screen Capture ──────────────────────────────────────────────────────────
+/// Capture from screen, or load from file if specified.
+/// When file is set, loads the image and crops to region.
+Bitmap CaptureOrLoad(string? file, RegionRect? region)
+{
+ if (!string.IsNullOrEmpty(file))
+ {
+ var fullBmp = new Bitmap(file);
+ if (region != null)
+ {
+ int cx = Math.Max(0, region.X);
+ int cy = Math.Max(0, region.Y);
+ int cw = Math.Min(region.Width, fullBmp.Width - cx);
+ int ch = Math.Min(region.Height, fullBmp.Height - cy);
+ var cropped = fullBmp.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb);
+ fullBmp.Dispose();
+ return cropped;
+ }
+ return fullBmp;
+ }
+ return CaptureScreen(region);
+}
+
Bitmap CaptureScreen(RegionRect? region)
{
int x, y, w, h;
@@ -203,6 +767,27 @@ class Request
[JsonPropertyName("path")]
public string? Path { get; set; }
+
+ [JsonPropertyName("cols")]
+ public int Cols { get; set; }
+
+ [JsonPropertyName("rows")]
+ public int Rows { get; set; }
+
+ [JsonPropertyName("threshold")]
+ public int Threshold { get; set; }
+
+ [JsonPropertyName("minCellSize")]
+ public int MinCellSize { get; set; }
+
+ [JsonPropertyName("maxCellSize")]
+ public int MaxCellSize { get; set; }
+
+ [JsonPropertyName("file")]
+ public string? File { get; set; }
+
+ [JsonPropertyName("debug")]
+ public bool Debug { get; set; }
}
class RegionRect
@@ -291,3 +876,36 @@ class CaptureResponse
[JsonPropertyName("image")]
public string Image { get; set; } = "";
}
+
+class GridResponse
+{
+ [JsonPropertyName("ok")]
+ public bool Ok => true;
+
+ [JsonPropertyName("cells")]
+ public List> Cells { get; set; } = [];
+}
+
+class DetectGridResponse
+{
+ [JsonPropertyName("ok")]
+ public bool Ok => true;
+
+ [JsonPropertyName("detected")]
+ public bool Detected { get; set; }
+
+ [JsonPropertyName("region")]
+ public RegionRect? Region { get; set; }
+
+ [JsonPropertyName("cols")]
+ public int Cols { get; set; }
+
+ [JsonPropertyName("rows")]
+ public int Rows { get; set; }
+
+ [JsonPropertyName("cellWidth")]
+ public double CellWidth { get; set; }
+
+ [JsonPropertyName("cellHeight")]
+ public double CellHeight { get; set; }
+}
`;
+ }
+ html += `