357 lines
15 KiB
C#
357 lines
15 KiB
C#
namespace OcrDaemon;
|
|
|
|
using System.Drawing;
|
|
using System.Drawing.Imaging;
|
|
using System.Runtime.InteropServices;
|
|
|
|
class GridHandler
|
|
{
|
|
// Pre-loaded empty cell templates (loaded lazily on first grid scan)
|
|
private byte[]? _emptyTemplate70Gray;
|
|
private byte[]? _emptyTemplate70Argb;
|
|
private int _emptyTemplate70W, _emptyTemplate70H, _emptyTemplate70Stride;
|
|
private byte[]? _emptyTemplate35Gray;
|
|
private byte[]? _emptyTemplate35Argb;
|
|
private int _emptyTemplate35W, _emptyTemplate35H, _emptyTemplate35Stride;
|
|
|
|
public object HandleGrid(Request req)
|
|
{
|
|
if (req.Region == null || req.Cols <= 0 || req.Rows <= 0)
|
|
return new ErrorResponse("grid command requires region, cols, rows");
|
|
|
|
LoadTemplatesIfNeeded();
|
|
|
|
using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region);
|
|
int cols = req.Cols;
|
|
int rows = req.Rows;
|
|
float cellW = (float)bitmap.Width / cols;
|
|
float cellH = (float)bitmap.Height / rows;
|
|
|
|
// Pick the right empty template based on cell size
|
|
int nominalCell = (int)Math.Round(cellW);
|
|
byte[]? templateGray;
|
|
byte[]? templateArgb;
|
|
int templateW, templateH, templateStride;
|
|
if (nominalCell <= 40 && _emptyTemplate35Gray != null)
|
|
{
|
|
templateGray = _emptyTemplate35Gray;
|
|
templateArgb = _emptyTemplate35Argb!;
|
|
templateW = _emptyTemplate35W;
|
|
templateH = _emptyTemplate35H;
|
|
templateStride = _emptyTemplate35Stride;
|
|
}
|
|
else if (_emptyTemplate70Gray != null)
|
|
{
|
|
templateGray = _emptyTemplate70Gray;
|
|
templateArgb = _emptyTemplate70Argb!;
|
|
templateW = _emptyTemplate70W;
|
|
templateH = _emptyTemplate70H;
|
|
templateStride = _emptyTemplate70Stride;
|
|
}
|
|
else
|
|
{
|
|
return new ErrorResponse("Empty cell templates not found in assets/");
|
|
}
|
|
|
|
// Convert captured bitmap to grayscale + keep ARGB for border color comparison
|
|
var (captureGray, captureArgb, captureStride) = ImageUtils.BitmapToGrayAndArgb(bitmap);
|
|
int captureW = bitmap.Width;
|
|
|
|
// Border to skip (outer pixels may differ between cells)
|
|
int border = Math.Max(2, nominalCell / 10);
|
|
|
|
// Pre-compute template average for the inner region
|
|
long templateSum = 0;
|
|
int innerCount = 0;
|
|
for (int ty = border; ty < templateH - border; ty++)
|
|
for (int tx = border; tx < templateW - border; tx++)
|
|
{
|
|
templateSum += templateGray[ty * templateW + tx];
|
|
innerCount++;
|
|
}
|
|
double tmplMean = innerCount > 0 ? (double)templateSum / innerCount : 0;
|
|
|
|
// Threshold for brightness-normalized MAD
|
|
double diffThreshold = req.Threshold > 0 ? req.Threshold : 5;
|
|
bool debug = req.Debug;
|
|
|
|
if (debug) Console.Error.WriteLine($"Grid: {cols}x{rows}, cellW={cellW:F1}, cellH={cellH:F1}, border={border}, threshold={diffThreshold}, tmplMean={tmplMean:F1}");
|
|
|
|
var cells = new List<List<bool>>();
|
|
for (int row = 0; row < rows; row++)
|
|
{
|
|
var rowList = new List<bool>();
|
|
var debugDiffs = new List<string>();
|
|
for (int col = 0; col < cols; col++)
|
|
{
|
|
int cx0 = (int)(col * cellW);
|
|
int cy0 = (int)(row * cellH);
|
|
int cw = (int)Math.Min(cellW, captureW - cx0);
|
|
int ch = (int)Math.Min(cellH, bitmap.Height - cy0);
|
|
|
|
int innerW = Math.Min(cw, templateW) - border;
|
|
int innerH = Math.Min(ch, templateH) - border;
|
|
|
|
// First pass: compute cell region mean brightness
|
|
long cellSum = 0;
|
|
int compared = 0;
|
|
for (int py = border; py < innerH; py++)
|
|
for (int px = border; px < innerW; px++)
|
|
{
|
|
cellSum += captureGray[(cy0 + py) * captureW + (cx0 + px)];
|
|
compared++;
|
|
}
|
|
double cellMean = compared > 0 ? (double)cellSum / compared : 0;
|
|
double offset = cellMean - tmplMean;
|
|
|
|
// Second pass: MAD on brightness-normalized values
|
|
long diffSum = 0;
|
|
for (int py = border; py < innerH; py++)
|
|
for (int px = border; px < innerW; px++)
|
|
{
|
|
double cellVal = captureGray[(cy0 + py) * captureW + (cx0 + px)];
|
|
double tmplVal = templateGray[py * templateW + px];
|
|
diffSum += (long)Math.Abs(cellVal - tmplVal - offset);
|
|
}
|
|
double meanDiff = compared > 0 ? (double)diffSum / compared : 0;
|
|
bool occupied = meanDiff > diffThreshold;
|
|
rowList.Add(occupied);
|
|
if (debug) debugDiffs.Add($"{meanDiff,5:F1}{(occupied ? "*" : " ")}");
|
|
}
|
|
cells.Add(rowList);
|
|
if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}");
|
|
}
|
|
|
|
// ── Item detection: compare border pixels to empty template (grayscale) ──
|
|
// Items have a colored tint behind them that shows through grid lines.
|
|
// Compare each cell's border strip against the template's border pixels.
|
|
// If they differ → item tint present → cells belong to same item.
|
|
int[] parent = new int[rows * cols];
|
|
for (int i = 0; i < parent.Length; i++) parent[i] = i;
|
|
|
|
int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; }
|
|
void Union(int a, int b) { parent[Find(a)] = Find(b); }
|
|
|
|
int stripWidth = Math.Max(2, border / 2);
|
|
int stripInset = (int)(cellW * 0.15);
|
|
double borderDiffThresh = 15.0;
|
|
|
|
for (int row = 0; row < rows; row++)
|
|
{
|
|
for (int col = 0; col < cols; col++)
|
|
{
|
|
if (!cells[row][col]) continue;
|
|
int cx0 = (int)(col * cellW);
|
|
int cy0 = (int)(row * cellH);
|
|
|
|
// Check right neighbor
|
|
if (col + 1 < cols && cells[row][col + 1])
|
|
{
|
|
long diffSum = 0; int cnt = 0;
|
|
int xStart = (int)((col + 1) * cellW) - stripWidth;
|
|
int yFrom = cy0 + stripInset;
|
|
int yTo = (int)((row + 1) * cellH) - stripInset;
|
|
for (int sy = yFrom; sy < yTo; sy += 2)
|
|
{
|
|
int tmplY = sy - cy0;
|
|
for (int sx = xStart; sx < xStart + stripWidth * 2; sx++)
|
|
{
|
|
if (sx < 0 || sx >= captureW) continue;
|
|
int tmplX = sx - cx0;
|
|
if (tmplX < 0 || tmplX >= templateW) continue;
|
|
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
|
cnt++;
|
|
}
|
|
}
|
|
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
|
if (debug) Console.Error.WriteLine($" H ({row},{col})->({row},{col+1}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
|
if (meanDiff > borderDiffThresh)
|
|
Union(row * cols + col, row * cols + col + 1);
|
|
}
|
|
|
|
// Check bottom neighbor
|
|
if (row + 1 < rows && cells[row + 1][col])
|
|
{
|
|
long diffSum = 0; int cnt = 0;
|
|
int yStart = (int)((row + 1) * cellH) - stripWidth;
|
|
int xFrom = cx0 + stripInset;
|
|
int xTo = (int)((col + 1) * cellW) - stripInset;
|
|
for (int sx = xFrom; sx < xTo; sx += 2)
|
|
{
|
|
int tmplX = sx - cx0;
|
|
for (int sy = yStart; sy < yStart + stripWidth * 2; sy++)
|
|
{
|
|
if (sy < 0 || sy >= bitmap.Height) continue;
|
|
int tmplY = sy - cy0;
|
|
if (tmplY < 0 || tmplY >= templateH) continue;
|
|
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
|
cnt++;
|
|
}
|
|
}
|
|
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
|
if (debug) Console.Error.WriteLine($" V ({row},{col})->({row+1},{col}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
|
if (meanDiff > borderDiffThresh)
|
|
Union(row * cols + col, (row + 1) * cols + col);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Extract items from union-find groups
|
|
var groups = new Dictionary<int, List<(int row, int col)>>();
|
|
for (int row = 0; row < rows; row++)
|
|
for (int col = 0; col < cols; col++)
|
|
if (cells[row][col])
|
|
{
|
|
int root = Find(row * cols + col);
|
|
if (!groups.ContainsKey(root)) groups[root] = [];
|
|
groups[root].Add((row, col));
|
|
}
|
|
|
|
var items = new List<GridItem>();
|
|
foreach (var group in groups.Values)
|
|
{
|
|
int minR = group.Min(c => c.row);
|
|
int maxR = group.Max(c => c.row);
|
|
int minC = group.Min(c => c.col);
|
|
int maxC = group.Max(c => c.col);
|
|
items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 });
|
|
}
|
|
|
|
if (debug)
|
|
{
|
|
Console.Error.WriteLine($" Items found: {items.Count}");
|
|
foreach (var item in items)
|
|
Console.Error.WriteLine($" ({item.Row},{item.Col}) {item.W}x{item.H}");
|
|
}
|
|
|
|
// ── Visual matching: find cells similar to target ──
|
|
List<GridMatch>? matches = null;
|
|
if (req.TargetRow >= 0 && req.TargetCol >= 0 &&
|
|
req.TargetRow < rows && req.TargetCol < cols &&
|
|
cells[req.TargetRow][req.TargetCol])
|
|
{
|
|
matches = FindMatchingCells(
|
|
captureGray, captureW, bitmap.Height,
|
|
cells, rows, cols, cellW, cellH, border,
|
|
req.TargetRow, req.TargetCol, debug);
|
|
}
|
|
|
|
return new GridResponse { Cells = cells, Items = items, Matches = matches };
|
|
}
|
|
|
|
/// <summary>
|
|
/// Find all occupied cells visually similar to the target cell using full-resolution NCC.
|
|
/// </summary>
|
|
private List<GridMatch> FindMatchingCells(
|
|
byte[] gray, int imgW, int imgH,
|
|
List<List<bool>> cells, int rows, int cols,
|
|
float cellW, float cellH, int border,
|
|
int targetRow, int targetCol, bool debug)
|
|
{
|
|
int innerW = (int)cellW - border * 2;
|
|
int innerH = (int)cellH - border * 2;
|
|
if (innerW <= 4 || innerH <= 4) return [];
|
|
|
|
int tCx0 = (int)(targetCol * cellW) + border;
|
|
int tCy0 = (int)(targetRow * cellH) + border;
|
|
int tInnerW = Math.Min(innerW, imgW - tCx0);
|
|
int tInnerH = Math.Min(innerH, imgH - tCy0);
|
|
if (tInnerW < innerW || tInnerH < innerH) return [];
|
|
|
|
int n = innerW * innerH;
|
|
|
|
// Pre-compute target cell pixels and stats
|
|
double[] targetPixels = new double[n];
|
|
double tMean = 0;
|
|
for (int py = 0; py < innerH; py++)
|
|
for (int px = 0; px < innerW; px++)
|
|
{
|
|
double v = gray[(tCy0 + py) * imgW + (tCx0 + px)];
|
|
targetPixels[py * innerW + px] = v;
|
|
tMean += v;
|
|
}
|
|
tMean /= n;
|
|
|
|
double tStd = 0;
|
|
for (int i = 0; i < n; i++)
|
|
tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean);
|
|
tStd = Math.Sqrt(tStd / n);
|
|
|
|
if (debug) Console.Error.WriteLine($" Match target ({targetRow},{targetCol}): {innerW}x{innerH} ({n}px), mean={tMean:F1}, std={tStd:F1}");
|
|
if (tStd < 3.0) return [];
|
|
|
|
double matchThreshold = 0.70;
|
|
var matches = new List<GridMatch>();
|
|
|
|
for (int row = 0; row < rows; row++)
|
|
{
|
|
for (int col = 0; col < cols; col++)
|
|
{
|
|
if (!cells[row][col]) continue;
|
|
if (row == targetRow && col == targetCol) continue;
|
|
|
|
int cx0 = (int)(col * cellW) + border;
|
|
int cy0 = (int)(row * cellH) + border;
|
|
int cInnerW = Math.Min(innerW, imgW - cx0);
|
|
int cInnerH = Math.Min(innerH, imgH - cy0);
|
|
if (cInnerW < innerW || cInnerH < innerH) continue;
|
|
|
|
// Compute NCC at full resolution
|
|
double cMean = 0;
|
|
for (int py = 0; py < innerH; py++)
|
|
for (int px = 0; px < innerW; px++)
|
|
cMean += gray[(cy0 + py) * imgW + (cx0 + px)];
|
|
cMean /= n;
|
|
|
|
double cStd = 0, cross = 0;
|
|
for (int py = 0; py < innerH; py++)
|
|
for (int px = 0; px < innerW; px++)
|
|
{
|
|
double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean;
|
|
double tv = targetPixels[py * innerW + px] - tMean;
|
|
cStd += cv * cv;
|
|
cross += tv * cv;
|
|
}
|
|
cStd = Math.Sqrt(cStd / n);
|
|
|
|
double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0;
|
|
|
|
if (debug && ncc > 0.5)
|
|
Console.Error.WriteLine($" ({row},{col}): NCC={ncc:F3}");
|
|
|
|
if (ncc >= matchThreshold)
|
|
matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) });
|
|
}
|
|
}
|
|
|
|
if (debug) Console.Error.WriteLine($" Matches for ({targetRow},{targetCol}): {matches.Count}");
|
|
return matches;
|
|
}
|
|
|
|
private void LoadTemplatesIfNeeded()
|
|
{
|
|
if (_emptyTemplate70Gray != null) return;
|
|
|
|
// Look for templates relative to exe directory
|
|
var exeDir = AppContext.BaseDirectory;
|
|
// Templates are in assets/ at project root — walk up from bin/Release/net8.0-.../
|
|
var projectRoot = Path.GetFullPath(Path.Combine(exeDir, "..", "..", "..", "..", ".."));
|
|
var t70Path = Path.Combine(projectRoot, "assets", "empty70.png");
|
|
var t35Path = Path.Combine(projectRoot, "assets", "empty35.png");
|
|
|
|
if (File.Exists(t70Path))
|
|
{
|
|
using var bmp = new Bitmap(t70Path);
|
|
_emptyTemplate70W = bmp.Width;
|
|
_emptyTemplate70H = bmp.Height;
|
|
(_emptyTemplate70Gray, _emptyTemplate70Argb, _emptyTemplate70Stride) = ImageUtils.BitmapToGrayAndArgb(bmp);
|
|
}
|
|
if (File.Exists(t35Path))
|
|
{
|
|
using var bmp = new Bitmap(t35Path);
|
|
_emptyTemplate35W = bmp.Width;
|
|
_emptyTemplate35H = bmp.Height;
|
|
(_emptyTemplate35Gray, _emptyTemplate35Argb, _emptyTemplate35Stride) = ImageUtils.BitmapToGrayAndArgb(bmp);
|
|
}
|
|
}
|
|
}
|