finished up grid detection.

This commit is contained in:
Boki 2026-02-10 17:29:37 -05:00
parent 41d174195e
commit 1246884be9
11 changed files with 1037 additions and 19 deletions

View file

@ -57,6 +57,12 @@ while ((line = stdin.ReadLine()) != null)
case "capture":
HandleCapture(request);
break;
case "grid":
HandleGrid(request);
break;
case "detect-grid":
HandleDetectGrid(request);
break;
default:
WriteResponse(new ErrorResponse($"Unknown command: {request.Cmd}"));
break;
@ -74,7 +80,7 @@ return 0;
void HandleOcr(Request req, OcrEngine engine)
{
using var bitmap = CaptureScreen(req.Region);
using var bitmap = CaptureOrLoad(req.File, req.Region);
var softwareBitmap = BitmapToSoftwareBitmap(bitmap);
var result = engine.RecognizeAsync(softwareBitmap).AsTask().GetAwaiter().GetResult();
@ -107,7 +113,7 @@ void HandleScreenshot(Request req)
return;
}
using var bitmap = CaptureScreen(req.Region);
using var bitmap = CaptureOrLoad(req.File, req.Region);
var format = GetImageFormat(req.Path);
bitmap.Save(req.Path, format);
WriteResponse(new OkResponse());
@ -115,15 +121,573 @@ void HandleScreenshot(Request req)
void HandleCapture(Request req)
{
using var bitmap = CaptureScreen(req.Region);
using var bitmap = CaptureOrLoad(req.File, req.Region);
using var ms = new MemoryStream();
bitmap.Save(ms, ImageFormat.Png);
var base64 = Convert.ToBase64String(ms.ToArray());
WriteResponse(new CaptureResponse { Image = base64 });
}
// Pre-loaded empty cell templates (loaded lazily on first grid scan)
byte[]? emptyTemplate70Gray = null;
int emptyTemplate70W = 0, emptyTemplate70H = 0;
byte[]? emptyTemplate35Gray = null;
int emptyTemplate35W = 0, emptyTemplate35H = 0;
void LoadTemplatesIfNeeded()
{
if (emptyTemplate70Gray != null) return;
// Look for templates relative to exe directory
var exeDir = AppContext.BaseDirectory;
// Templates are in assets/ at project root — walk up from bin/Release/net8.0-.../
var projectRoot = System.IO.Path.GetFullPath(System.IO.Path.Combine(exeDir, "..", "..", "..", "..", ".."));
var t70Path = System.IO.Path.Combine(projectRoot, "assets", "empty70.png");
var t35Path = System.IO.Path.Combine(projectRoot, "assets", "empty35.png");
if (System.IO.File.Exists(t70Path))
{
using var bmp = new Bitmap(t70Path);
emptyTemplate70W = bmp.Width;
emptyTemplate70H = bmp.Height;
emptyTemplate70Gray = BitmapToGray(bmp);
}
if (System.IO.File.Exists(t35Path))
{
using var bmp = new Bitmap(t35Path);
emptyTemplate35W = bmp.Width;
emptyTemplate35H = bmp.Height;
emptyTemplate35Gray = BitmapToGray(bmp);
}
}
byte[] BitmapToGray(Bitmap bmp)
{
int w = bmp.Width, h = bmp.Height;
var data = bmp.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
byte[] pixels = new byte[data.Stride * h];
Marshal.Copy(data.Scan0, pixels, 0, pixels.Length);
bmp.UnlockBits(data);
int stride = data.Stride;
byte[] gray = new byte[w * h];
for (int y = 0; y < h; y++)
for (int x = 0; x < w; x++)
{
int i = y * stride + x * 4;
gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
}
return gray;
}
void HandleGrid(Request req)
{
if (req.Region == null || req.Cols <= 0 || req.Rows <= 0)
{
WriteResponse(new ErrorResponse("grid command requires region, cols, rows"));
return;
}
LoadTemplatesIfNeeded();
using var bitmap = CaptureOrLoad(req.File, req.Region);
int cols = req.Cols;
int rows = req.Rows;
float cellW = (float)bitmap.Width / cols;
float cellH = (float)bitmap.Height / rows;
// Pick the right empty template based on cell size
int nominalCell = (int)Math.Round(cellW);
byte[]? templateGray;
int templateW, templateH;
if (nominalCell <= 40 && emptyTemplate35Gray != null)
{
templateGray = emptyTemplate35Gray;
templateW = emptyTemplate35W;
templateH = emptyTemplate35H;
}
else if (emptyTemplate70Gray != null)
{
templateGray = emptyTemplate70Gray;
templateW = emptyTemplate70W;
templateH = emptyTemplate70H;
}
else
{
WriteResponse(new ErrorResponse("Empty cell templates not found in assets/"));
return;
}
// Convert captured bitmap to grayscale
byte[] captureGray = BitmapToGray(bitmap);
int captureW = bitmap.Width;
// Border to skip (outer pixels may differ between cells)
int border = Math.Max(2, nominalCell / 10);
// Pre-compute template average for the inner region
long templateSum = 0;
int innerCount = 0;
for (int ty = border; ty < templateH - border; ty++)
for (int tx = border; tx < templateW - border; tx++)
{
templateSum += templateGray[ty * templateW + tx];
innerCount++;
}
// Threshold for mean absolute difference — default 6
double diffThreshold = req.Threshold > 0 ? req.Threshold : 2;
bool debug = req.Debug;
if (debug) Console.Error.WriteLine($"Grid: {cols}x{rows}, cellW={cellW:F1}, cellH={cellH:F1}, border={border}, threshold={diffThreshold}");
var cells = new List<List<bool>>();
for (int row = 0; row < rows; row++)
{
var rowList = new List<bool>();
var debugDiffs = new List<string>();
for (int col = 0; col < cols; col++)
{
int cx0 = (int)(col * cellW);
int cy0 = (int)(row * cellH);
int cw = (int)Math.Min(cellW, captureW - cx0);
int ch = (int)Math.Min(cellH, bitmap.Height - cy0);
// Compare inner pixels of cell vs template
long diffSum = 0;
int compared = 0;
int innerW = Math.Min(cw, templateW) - border;
int innerH = Math.Min(ch, templateH) - border;
for (int py = border; py < innerH; py++)
{
for (int px = border; px < innerW; px++)
{
int cellVal = captureGray[(cy0 + py) * captureW + (cx0 + px)];
int tmplVal = templateGray[py * templateW + px];
diffSum += Math.Abs(cellVal - tmplVal);
compared++;
}
}
double meanDiff = compared > 0 ? (double)diffSum / compared : 0;
bool occupied = meanDiff > diffThreshold;
rowList.Add(occupied);
if (debug) debugDiffs.Add($"{meanDiff,5:F1}{(occupied ? "*" : " ")}");
}
cells.Add(rowList);
if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}");
}
WriteResponse(new GridResponse { Cells = cells });
}
void HandleDetectGrid(Request req)
{
if (req.Region == null)
{
WriteResponse(new ErrorResponse("detect-grid requires region"));
return;
}
int minCell = req.MinCellSize > 0 ? req.MinCellSize : 20;
int maxCell = req.MaxCellSize > 0 ? req.MaxCellSize : 70;
bool debug = req.Debug;
Bitmap bitmap = CaptureOrLoad(req.File, req.Region);
int w = bitmap.Width;
int h = bitmap.Height;
var bmpData = bitmap.LockBits(
new Rectangle(0, 0, w, h),
ImageLockMode.ReadOnly,
PixelFormat.Format32bppArgb
);
byte[] pixels = new byte[bmpData.Stride * h];
Marshal.Copy(bmpData.Scan0, pixels, 0, pixels.Length);
bitmap.UnlockBits(bmpData);
int stride = bmpData.Stride;
byte[] gray = new byte[w * h];
for (int y = 0; y < h; y++)
for (int x = 0; x < w; x++)
{
int i = y * stride + x * 4;
gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
}
bitmap.Dispose();
// ── Pass 1: Scan horizontal bands using "very dark pixel density" ──
// Grid lines are nearly all very dark (density ~0.9), cell interiors are
// partially dark (0.3-0.5), game world is mostly bright (density ~0.05).
// This creates clear periodic peaks at grid line positions.
int bandH = 200;
int bandStep = 40;
const int veryDarkPixelThresh = 12; // pixels below this brightness = "very dark"
const double gridSegThresh = 0.25; // density above this = potential grid column
var candidates = new List<(int bandY, int cellW, double hAc, int hLeft, int hRight)>();
for (int by = 0; by + bandH <= h; by += bandStep)
{
// "Very dark pixel density" per column: fraction of pixels below threshold
double[] darkDensity = new double[w];
for (int x = 0; x < w; x++)
{
int count = 0;
for (int y = by; y < by + bandH; y++)
{
if (gray[y * w + x] < veryDarkPixelThresh) count++;
}
darkDensity[x] = (double)count / bandH;
}
// Find segments where density > gridSegThresh (grid panel regions)
var gridSegs = FindDarkDensitySegments(darkDensity, gridSegThresh, 200);
foreach (var (segLeft, segRight) in gridSegs)
{
// Extract segment and run AC
int segLen = segRight - segLeft;
double[] segment = new double[segLen];
Array.Copy(darkDensity, segLeft, segment, 0, segLen);
var (period, acScore) = FindPeriodWithScore(segment, minCell, maxCell);
if (period <= 0) continue;
// FindGridExtent within the segment
var (extLeft, extRight) = FindGridExtent(segment, period);
if (extLeft < 0) continue;
// Map back to full image coordinates
int absLeft = segLeft + extLeft;
int absRight = segLeft + extRight;
int extent = absRight - absLeft;
// Require at least 8 cells wide AND 200px absolute minimum
if (extent < period * 8 || extent < 200) continue;
if (debug) Console.Error.WriteLine(
$" Band y={by}: seg=[{segLeft}-{segRight}] period={period}, AC={acScore:F3}, " +
$"extent={absLeft}-{absRight}={extent}px ({extent / period} cells)");
candidates.Add((by, period, acScore, absLeft, absRight));
}
}
if (debug) Console.Error.WriteLine($"Pass 1: {candidates.Count} candidates");
// Sort by score = AC * extent (prefer large strongly-periodic areas)
candidates.Sort((a, b) =>
{
double sa = a.hAc * (a.hRight - a.hLeft);
double sb = b.hAc * (b.hRight - b.hLeft);
return sb.CompareTo(sa);
});
// ── Pass 2: Verify vertical periodicity ──
foreach (var cand in candidates.Take(10))
{
int colSpan = cand.hRight - cand.hLeft;
if (colSpan < cand.cellW * 3) continue;
// Row "very dark pixel density" within the detected column range
double[] rowDensity = new double[h];
for (int y = 0; y < h; y++)
{
int count = 0;
for (int x = cand.hLeft; x < cand.hRight; x++)
{
if (gray[y * w + x] < veryDarkPixelThresh) count++;
}
rowDensity[y] = (double)count / colSpan;
}
// Find grid panel vertical segment
var vGridSegs = FindDarkDensitySegments(rowDensity, gridSegThresh, 100);
if (vGridSegs.Count == 0) continue;
// Use the largest segment
var (vSegTop, vSegBottom) = vGridSegs.OrderByDescending(s => s.end - s.start).First();
int vSegLen = vSegBottom - vSegTop;
double[] vSegment = new double[vSegLen];
Array.Copy(rowDensity, vSegTop, vSegment, 0, vSegLen);
var (cellH, vAc) = FindPeriodWithScore(vSegment, minCell, maxCell);
if (cellH <= 0) continue;
var (extTop, extBottom) = FindGridExtent(vSegment, cellH);
if (extTop < 0) continue;
int top = vSegTop + extTop;
int bottom = vSegTop + extBottom;
int vExtent = bottom - top;
// Require at least 3 rows tall AND 100px absolute minimum
if (vExtent < cellH * 3 || vExtent < 100) continue;
if (debug) Console.Error.WriteLine(
$" 2D candidate: cellW={cand.cellW}, cellH={cellH}, " +
$"region=({cand.hLeft},{top})-({cand.hRight},{bottom}), " +
$"vAC={vAc:F3}, extent={vExtent}px ({vExtent / cellH} rows)");
// ── Found a valid 2D grid ──
int gridW = cand.hRight - cand.hLeft;
int gridH = bottom - top;
int cols = Math.Max(2, (int)Math.Round((double)gridW / cand.cellW));
int rows = Math.Max(2, (int)Math.Round((double)gridH / cellH));
// Snap grid dimensions to exact multiples of cell size
gridW = cols * cand.cellW;
gridH = rows * cellH;
if (debug) Console.Error.WriteLine(
$" => cols={cols}, rows={rows}, gridW={gridW}, gridH={gridH}");
WriteResponse(new DetectGridResponse
{
Detected = true,
Region = new RegionRect
{
X = req.Region.X + cand.hLeft,
Y = req.Region.Y + top,
Width = gridW,
Height = gridH,
},
Cols = cols,
Rows = rows,
CellWidth = Math.Round((double)gridW / cols, 1),
CellHeight = Math.Round((double)gridH / rows, 1),
});
return;
}
if (debug) Console.Error.WriteLine(" No valid 2D grid found");
WriteResponse(new DetectGridResponse { Detected = false });
}
/// Find the dominant period in a signal using autocorrelation.
/// Returns (period, score) where score is the autocorrelation strength.
(int period, double score) FindPeriodWithScore(double[] signal, int minPeriod, int maxPeriod)
{
int n = signal.Length;
if (n < minPeriod * 3) return (-1, 0);
double mean = signal.Average();
double variance = 0;
for (int i = 0; i < n; i++)
variance += (signal[i] - mean) * (signal[i] - mean);
if (variance < 1.0) return (-1, 0);
int maxLag = Math.Min(maxPeriod, n / 3);
double[] ac = new double[maxLag + 1];
for (int lag = minPeriod; lag <= maxLag; lag++)
{
double sum = 0;
for (int i = 0; i < n - lag; i++)
sum += (signal[i] - mean) * (signal[i + lag] - mean);
ac[lag] = sum / variance;
}
// Find the first significant peak — this is the fundamental period.
// Using "first" avoids picking harmonics (2x, 3x) or unrelated larger patterns.
for (int lag = minPeriod + 1; lag < maxLag; lag++)
{
if (ac[lag] > 0.01 && ac[lag] >= ac[lag - 1] && ac[lag] >= ac[lag + 1])
return (lag, ac[lag]);
}
return (-1, 0);
}
/// Find contiguous segments where values are ABOVE threshold.
/// Used to find grid panel regions by density of very dark pixels.
/// Allows brief gaps (up to 5px) to handle grid borders.
List<(int start, int end)> FindDarkDensitySegments(double[] profile, double threshold, int minLength)
{
var segments = new List<(int start, int end)>();
int n = profile.Length;
int curStart = -1;
int maxGap = 5;
int gapCount = 0;
for (int i = 0; i < n; i++)
{
if (profile[i] >= threshold)
{
if (curStart < 0) curStart = i;
gapCount = 0;
}
else
{
if (curStart >= 0)
{
gapCount++;
if (gapCount > maxGap)
{
int end = i - gapCount;
if (end - curStart >= minLength)
segments.Add((curStart, end));
curStart = -1;
gapCount = 0;
}
}
}
}
if (curStart >= 0)
{
int end = gapCount > 0 ? n - gapCount : n;
if (end - curStart >= minLength)
segments.Add((curStart, end));
}
return segments;
}
/// Debug: find the top N AC peaks in a signal
List<(int lag, double ac)> FindTopAcPeaks(double[] signal, int minPeriod, int maxPeriod, int topN)
{
int n = signal.Length;
if (n < minPeriod * 3) return [];
double mean = signal.Average();
double variance = 0;
for (int i = 0; i < n; i++)
variance += (signal[i] - mean) * (signal[i] - mean);
if (variance < 1.0) return [];
int maxLag = Math.Min(maxPeriod, n / 3);
var peaks = new List<(int lag, double ac)>();
double[] ac = new double[maxLag + 1];
for (int lag = minPeriod; lag <= maxLag; lag++)
{
double sum = 0;
for (int i = 0; i < n - lag; i++)
sum += (signal[i] - mean) * (signal[i + lag] - mean);
ac[lag] = sum / variance;
}
for (int lag = minPeriod + 1; lag < maxLag; lag++)
{
if (ac[lag] >= ac[lag - 1] && ac[lag] >= ac[lag + 1] && ac[lag] > 0.005)
peaks.Add((lag, ac[lag]));
}
peaks.Sort((a, b) => b.ac.CompareTo(a.ac));
return peaks.Take(topN).ToList();
}
/// Find the extent of the grid in a 1D profile using local autocorrelation
/// at the specific detected period. Only regions where the signal actually
/// repeats at the given period will score high — much more precise than variance.
(int start, int end) FindGridExtent(double[] signal, int period)
{
int n = signal.Length;
int halfWin = period * 2; // window radius: 2 periods each side
if (n < halfWin * 2 + period) return (-1, -1);
// Compute local AC at the specific lag=period in a sliding window
double[] localAc = new double[n];
for (int center = halfWin; center < n - halfWin; center++)
{
int wStart = center - halfWin;
int wEnd = center + halfWin;
int count = wEnd - wStart;
// Local mean
double sum = 0;
for (int i = wStart; i < wEnd; i++)
sum += signal[i];
double mean = sum / count;
// Local variance
double varSum = 0;
for (int i = wStart; i < wEnd; i++)
varSum += (signal[i] - mean) * (signal[i] - mean);
if (varSum < 1.0) continue;
// AC at the specific lag=period
double acSum = 0;
for (int i = wStart; i < wEnd - period; i++)
acSum += (signal[i] - mean) * (signal[i + period] - mean);
localAc[center] = Math.Max(0, acSum / varSum);
}
// Find the longest contiguous run above threshold
double maxAc = 0;
for (int i = 0; i < n; i++)
if (localAc[i] > maxAc) maxAc = localAc[i];
if (maxAc < 0.02) return (-1, -1);
double threshold = maxAc * 0.25;
int bestStart = -1, bestEnd = -1, bestLen = 0;
int curStart = -1;
for (int i = 0; i < n; i++)
{
if (localAc[i] > threshold)
{
if (curStart < 0) curStart = i;
}
else
{
if (curStart >= 0)
{
int len = i - curStart;
if (len > bestLen)
{
bestLen = len;
bestStart = curStart;
bestEnd = i;
}
curStart = -1;
}
}
}
// Handle run extending to end of signal
if (curStart >= 0)
{
int len = n - curStart;
if (len > bestLen)
{
bestStart = curStart;
bestEnd = n;
}
}
if (bestStart < 0) return (-1, -1);
// Small extension to include cell borders at edges
bestStart = Math.Max(0, bestStart - period / 4);
bestEnd = Math.Min(n - 1, bestEnd + period / 4);
return (bestStart, bestEnd);
}
// ── Screen Capture ──────────────────────────────────────────────────────────
/// Capture from screen, or load from file if specified.
/// When file is set, loads the image and crops to region.
Bitmap CaptureOrLoad(string? file, RegionRect? region)
{
if (!string.IsNullOrEmpty(file))
{
var fullBmp = new Bitmap(file);
if (region != null)
{
int cx = Math.Max(0, region.X);
int cy = Math.Max(0, region.Y);
int cw = Math.Min(region.Width, fullBmp.Width - cx);
int ch = Math.Min(region.Height, fullBmp.Height - cy);
var cropped = fullBmp.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb);
fullBmp.Dispose();
return cropped;
}
return fullBmp;
}
return CaptureScreen(region);
}
Bitmap CaptureScreen(RegionRect? region)
{
int x, y, w, h;
@ -203,6 +767,27 @@ class Request
[JsonPropertyName("path")]
public string? Path { get; set; }
[JsonPropertyName("cols")]
public int Cols { get; set; }
[JsonPropertyName("rows")]
public int Rows { get; set; }
[JsonPropertyName("threshold")]
public int Threshold { get; set; }
[JsonPropertyName("minCellSize")]
public int MinCellSize { get; set; }
[JsonPropertyName("maxCellSize")]
public int MaxCellSize { get; set; }
[JsonPropertyName("file")]
public string? File { get; set; }
[JsonPropertyName("debug")]
public bool Debug { get; set; }
}
class RegionRect
@ -291,3 +876,36 @@ class CaptureResponse
[JsonPropertyName("image")]
public string Image { get; set; } = "";
}
class GridResponse
{
[JsonPropertyName("ok")]
public bool Ok => true;
[JsonPropertyName("cells")]
public List<List<bool>> Cells { get; set; } = [];
}
class DetectGridResponse
{
[JsonPropertyName("ok")]
public bool Ok => true;
[JsonPropertyName("detected")]
public bool Detected { get; set; }
[JsonPropertyName("region")]
public RegionRect? Region { get; set; }
[JsonPropertyName("cols")]
public int Cols { get; set; }
[JsonPropertyName("rows")]
public int Rows { get; set; }
[JsonPropertyName("cellWidth")]
public double CellWidth { get; set; }
[JsonPropertyName("cellHeight")]
public double CellHeight { get; set; }
}