finished ocr daemon update
This commit is contained in:
parent
cc50368d3b
commit
6ad382cb09
13 changed files with 1471 additions and 1479 deletions
347
tools/OcrDaemon/GridHandler.cs
Normal file
347
tools/OcrDaemon/GridHandler.cs
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
namespace OcrDaemon;
|
||||
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
class GridHandler
|
||||
{
|
||||
// Pre-loaded empty cell templates (loaded lazily on first grid scan)
|
||||
private byte[]? _emptyTemplate70Gray;
|
||||
private byte[]? _emptyTemplate70Argb;
|
||||
private int _emptyTemplate70W, _emptyTemplate70H, _emptyTemplate70Stride;
|
||||
private byte[]? _emptyTemplate35Gray;
|
||||
private byte[]? _emptyTemplate35Argb;
|
||||
private int _emptyTemplate35W, _emptyTemplate35H, _emptyTemplate35Stride;
|
||||
|
||||
public object HandleGrid(Request req)
|
||||
{
|
||||
if (req.Region == null || req.Cols <= 0 || req.Rows <= 0)
|
||||
return new ErrorResponse("grid command requires region, cols, rows");
|
||||
|
||||
LoadTemplatesIfNeeded();
|
||||
|
||||
using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region);
|
||||
int cols = req.Cols;
|
||||
int rows = req.Rows;
|
||||
float cellW = (float)bitmap.Width / cols;
|
||||
float cellH = (float)bitmap.Height / rows;
|
||||
|
||||
// Pick the right empty template based on cell size
|
||||
int nominalCell = (int)Math.Round(cellW);
|
||||
byte[]? templateGray;
|
||||
byte[]? templateArgb;
|
||||
int templateW, templateH, templateStride;
|
||||
if (nominalCell <= 40 && _emptyTemplate35Gray != null)
|
||||
{
|
||||
templateGray = _emptyTemplate35Gray;
|
||||
templateArgb = _emptyTemplate35Argb!;
|
||||
templateW = _emptyTemplate35W;
|
||||
templateH = _emptyTemplate35H;
|
||||
templateStride = _emptyTemplate35Stride;
|
||||
}
|
||||
else if (_emptyTemplate70Gray != null)
|
||||
{
|
||||
templateGray = _emptyTemplate70Gray;
|
||||
templateArgb = _emptyTemplate70Argb!;
|
||||
templateW = _emptyTemplate70W;
|
||||
templateH = _emptyTemplate70H;
|
||||
templateStride = _emptyTemplate70Stride;
|
||||
}
|
||||
else
|
||||
{
|
||||
return new ErrorResponse("Empty cell templates not found in assets/");
|
||||
}
|
||||
|
||||
// Convert captured bitmap to grayscale + keep ARGB for border color comparison
|
||||
var (captureGray, captureArgb, captureStride) = ImageUtils.BitmapToGrayAndArgb(bitmap);
|
||||
int captureW = bitmap.Width;
|
||||
|
||||
// Border to skip (outer pixels may differ between cells)
|
||||
int border = Math.Max(2, nominalCell / 10);
|
||||
|
||||
// Pre-compute template average for the inner region
|
||||
long templateSum = 0;
|
||||
int innerCount = 0;
|
||||
for (int ty = border; ty < templateH - border; ty++)
|
||||
for (int tx = border; tx < templateW - border; tx++)
|
||||
{
|
||||
templateSum += templateGray[ty * templateW + tx];
|
||||
innerCount++;
|
||||
}
|
||||
|
||||
// Threshold for mean absolute difference — default 6
|
||||
double diffThreshold = req.Threshold > 0 ? req.Threshold : 2;
|
||||
bool debug = req.Debug;
|
||||
|
||||
if (debug) Console.Error.WriteLine($"Grid: {cols}x{rows}, cellW={cellW:F1}, cellH={cellH:F1}, border={border}, threshold={diffThreshold}");
|
||||
|
||||
var cells = new List<List<bool>>();
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
var rowList = new List<bool>();
|
||||
var debugDiffs = new List<string>();
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
int cx0 = (int)(col * cellW);
|
||||
int cy0 = (int)(row * cellH);
|
||||
int cw = (int)Math.Min(cellW, captureW - cx0);
|
||||
int ch = (int)Math.Min(cellH, bitmap.Height - cy0);
|
||||
|
||||
// Compare inner pixels of cell vs template
|
||||
long diffSum = 0;
|
||||
int compared = 0;
|
||||
int innerW = Math.Min(cw, templateW) - border;
|
||||
int innerH = Math.Min(ch, templateH) - border;
|
||||
for (int py = border; py < innerH; py++)
|
||||
{
|
||||
for (int px = border; px < innerW; px++)
|
||||
{
|
||||
int cellVal = captureGray[(cy0 + py) * captureW + (cx0 + px)];
|
||||
int tmplVal = templateGray[py * templateW + px];
|
||||
diffSum += Math.Abs(cellVal - tmplVal);
|
||||
compared++;
|
||||
}
|
||||
}
|
||||
double meanDiff = compared > 0 ? (double)diffSum / compared : 0;
|
||||
bool occupied = meanDiff > diffThreshold;
|
||||
rowList.Add(occupied);
|
||||
if (debug) debugDiffs.Add($"{meanDiff,5:F1}{(occupied ? "*" : " ")}");
|
||||
}
|
||||
cells.Add(rowList);
|
||||
if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}");
|
||||
}
|
||||
|
||||
// ── Item detection: compare border pixels to empty template (grayscale) ──
|
||||
// Items have a colored tint behind them that shows through grid lines.
|
||||
// Compare each cell's border strip against the template's border pixels.
|
||||
// If they differ → item tint present → cells belong to same item.
|
||||
int[] parent = new int[rows * cols];
|
||||
for (int i = 0; i < parent.Length; i++) parent[i] = i;
|
||||
|
||||
int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; }
|
||||
void Union(int a, int b) { parent[Find(a)] = Find(b); }
|
||||
|
||||
int stripWidth = Math.Max(2, border / 2);
|
||||
int stripInset = (int)(cellW * 0.15);
|
||||
double borderDiffThresh = 15.0;
|
||||
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
if (!cells[row][col]) continue;
|
||||
int cx0 = (int)(col * cellW);
|
||||
int cy0 = (int)(row * cellH);
|
||||
|
||||
// Check right neighbor
|
||||
if (col + 1 < cols && cells[row][col + 1])
|
||||
{
|
||||
long diffSum = 0; int cnt = 0;
|
||||
int xStart = (int)((col + 1) * cellW) - stripWidth;
|
||||
int yFrom = cy0 + stripInset;
|
||||
int yTo = (int)((row + 1) * cellH) - stripInset;
|
||||
for (int sy = yFrom; sy < yTo; sy += 2)
|
||||
{
|
||||
int tmplY = sy - cy0;
|
||||
for (int sx = xStart; sx < xStart + stripWidth * 2; sx++)
|
||||
{
|
||||
if (sx < 0 || sx >= captureW) continue;
|
||||
int tmplX = sx - cx0;
|
||||
if (tmplX < 0 || tmplX >= templateW) continue;
|
||||
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
||||
if (debug) Console.Error.WriteLine($" H ({row},{col})->({row},{col+1}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
||||
if (meanDiff > borderDiffThresh)
|
||||
Union(row * cols + col, row * cols + col + 1);
|
||||
}
|
||||
|
||||
// Check bottom neighbor
|
||||
if (row + 1 < rows && cells[row + 1][col])
|
||||
{
|
||||
long diffSum = 0; int cnt = 0;
|
||||
int yStart = (int)((row + 1) * cellH) - stripWidth;
|
||||
int xFrom = cx0 + stripInset;
|
||||
int xTo = (int)((col + 1) * cellW) - stripInset;
|
||||
for (int sx = xFrom; sx < xTo; sx += 2)
|
||||
{
|
||||
int tmplX = sx - cx0;
|
||||
for (int sy = yStart; sy < yStart + stripWidth * 2; sy++)
|
||||
{
|
||||
if (sy < 0 || sy >= bitmap.Height) continue;
|
||||
int tmplY = sy - cy0;
|
||||
if (tmplY < 0 || tmplY >= templateH) continue;
|
||||
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
||||
if (debug) Console.Error.WriteLine($" V ({row},{col})->({row+1},{col}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
||||
if (meanDiff > borderDiffThresh)
|
||||
Union(row * cols + col, (row + 1) * cols + col);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract items from union-find groups
|
||||
var groups = new Dictionary<int, List<(int row, int col)>>();
|
||||
for (int row = 0; row < rows; row++)
|
||||
for (int col = 0; col < cols; col++)
|
||||
if (cells[row][col])
|
||||
{
|
||||
int root = Find(row * cols + col);
|
||||
if (!groups.ContainsKey(root)) groups[root] = [];
|
||||
groups[root].Add((row, col));
|
||||
}
|
||||
|
||||
var items = new List<GridItem>();
|
||||
foreach (var group in groups.Values)
|
||||
{
|
||||
int minR = group.Min(c => c.row);
|
||||
int maxR = group.Max(c => c.row);
|
||||
int minC = group.Min(c => c.col);
|
||||
int maxC = group.Max(c => c.col);
|
||||
items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 });
|
||||
}
|
||||
|
||||
if (debug)
|
||||
{
|
||||
Console.Error.WriteLine($" Items found: {items.Count}");
|
||||
foreach (var item in items)
|
||||
Console.Error.WriteLine($" ({item.Row},{item.Col}) {item.W}x{item.H}");
|
||||
}
|
||||
|
||||
// ── Visual matching: find cells similar to target ──
|
||||
List<GridMatch>? matches = null;
|
||||
if (req.TargetRow >= 0 && req.TargetCol >= 0 &&
|
||||
req.TargetRow < rows && req.TargetCol < cols &&
|
||||
cells[req.TargetRow][req.TargetCol])
|
||||
{
|
||||
matches = FindMatchingCells(
|
||||
captureGray, captureW, bitmap.Height,
|
||||
cells, rows, cols, cellW, cellH, border,
|
||||
req.TargetRow, req.TargetCol, debug);
|
||||
}
|
||||
|
||||
return new GridResponse { Cells = cells, Items = items, Matches = matches };
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Find all occupied cells visually similar to the target cell using full-resolution NCC.
|
||||
/// </summary>
|
||||
private List<GridMatch> FindMatchingCells(
|
||||
byte[] gray, int imgW, int imgH,
|
||||
List<List<bool>> cells, int rows, int cols,
|
||||
float cellW, float cellH, int border,
|
||||
int targetRow, int targetCol, bool debug)
|
||||
{
|
||||
int innerW = (int)cellW - border * 2;
|
||||
int innerH = (int)cellH - border * 2;
|
||||
if (innerW <= 4 || innerH <= 4) return [];
|
||||
|
||||
int tCx0 = (int)(targetCol * cellW) + border;
|
||||
int tCy0 = (int)(targetRow * cellH) + border;
|
||||
int tInnerW = Math.Min(innerW, imgW - tCx0);
|
||||
int tInnerH = Math.Min(innerH, imgH - tCy0);
|
||||
if (tInnerW < innerW || tInnerH < innerH) return [];
|
||||
|
||||
int n = innerW * innerH;
|
||||
|
||||
// Pre-compute target cell pixels and stats
|
||||
double[] targetPixels = new double[n];
|
||||
double tMean = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
{
|
||||
double v = gray[(tCy0 + py) * imgW + (tCx0 + px)];
|
||||
targetPixels[py * innerW + px] = v;
|
||||
tMean += v;
|
||||
}
|
||||
tMean /= n;
|
||||
|
||||
double tStd = 0;
|
||||
for (int i = 0; i < n; i++)
|
||||
tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean);
|
||||
tStd = Math.Sqrt(tStd / n);
|
||||
|
||||
if (debug) Console.Error.WriteLine($" Match target ({targetRow},{targetCol}): {innerW}x{innerH} ({n}px), mean={tMean:F1}, std={tStd:F1}");
|
||||
if (tStd < 3.0) return [];
|
||||
|
||||
double matchThreshold = 0.70;
|
||||
var matches = new List<GridMatch>();
|
||||
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
if (!cells[row][col]) continue;
|
||||
if (row == targetRow && col == targetCol) continue;
|
||||
|
||||
int cx0 = (int)(col * cellW) + border;
|
||||
int cy0 = (int)(row * cellH) + border;
|
||||
int cInnerW = Math.Min(innerW, imgW - cx0);
|
||||
int cInnerH = Math.Min(innerH, imgH - cy0);
|
||||
if (cInnerW < innerW || cInnerH < innerH) continue;
|
||||
|
||||
// Compute NCC at full resolution
|
||||
double cMean = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
cMean += gray[(cy0 + py) * imgW + (cx0 + px)];
|
||||
cMean /= n;
|
||||
|
||||
double cStd = 0, cross = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
{
|
||||
double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean;
|
||||
double tv = targetPixels[py * innerW + px] - tMean;
|
||||
cStd += cv * cv;
|
||||
cross += tv * cv;
|
||||
}
|
||||
cStd = Math.Sqrt(cStd / n);
|
||||
|
||||
double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0;
|
||||
|
||||
if (debug && ncc > 0.5)
|
||||
Console.Error.WriteLine($" ({row},{col}): NCC={ncc:F3}");
|
||||
|
||||
if (ncc >= matchThreshold)
|
||||
matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) });
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) Console.Error.WriteLine($" Matches for ({targetRow},{targetCol}): {matches.Count}");
|
||||
return matches;
|
||||
}
|
||||
|
||||
private void LoadTemplatesIfNeeded()
|
||||
{
|
||||
if (_emptyTemplate70Gray != null) return;
|
||||
|
||||
// Look for templates relative to exe directory
|
||||
var exeDir = AppContext.BaseDirectory;
|
||||
// Templates are in assets/ at project root — walk up from bin/Release/net8.0-.../
|
||||
var projectRoot = Path.GetFullPath(Path.Combine(exeDir, "..", "..", "..", "..", ".."));
|
||||
var t70Path = Path.Combine(projectRoot, "assets", "empty70.png");
|
||||
var t35Path = Path.Combine(projectRoot, "assets", "empty35.png");
|
||||
|
||||
if (File.Exists(t70Path))
|
||||
{
|
||||
using var bmp = new Bitmap(t70Path);
|
||||
_emptyTemplate70W = bmp.Width;
|
||||
_emptyTemplate70H = bmp.Height;
|
||||
(_emptyTemplate70Gray, _emptyTemplate70Argb, _emptyTemplate70Stride) = ImageUtils.BitmapToGrayAndArgb(bmp);
|
||||
}
|
||||
if (File.Exists(t35Path))
|
||||
{
|
||||
using var bmp = new Bitmap(t35Path);
|
||||
_emptyTemplate35W = bmp.Width;
|
||||
_emptyTemplate35H = bmp.Height;
|
||||
(_emptyTemplate35Gray, _emptyTemplate35Argb, _emptyTemplate35Stride) = ImageUtils.BitmapToGrayAndArgb(bmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue