namespace OcrDaemon; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; class GridHandler { // Pre-loaded empty cell templates (loaded lazily on first grid scan) private byte[]? _emptyTemplate70Gray; private byte[]? _emptyTemplate70Argb; private int _emptyTemplate70W, _emptyTemplate70H, _emptyTemplate70Stride; private byte[]? _emptyTemplate35Gray; private byte[]? _emptyTemplate35Argb; private int _emptyTemplate35W, _emptyTemplate35H, _emptyTemplate35Stride; public object HandleGrid(Request req) { if (req.Region == null || req.Cols <= 0 || req.Rows <= 0) return new ErrorResponse("grid command requires region, cols, rows"); LoadTemplatesIfNeeded(); using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region); int cols = req.Cols; int rows = req.Rows; float cellW = (float)bitmap.Width / cols; float cellH = (float)bitmap.Height / rows; // Pick the right empty template based on cell size int nominalCell = (int)Math.Round(cellW); byte[]? templateGray; byte[]? templateArgb; int templateW, templateH, templateStride; if (nominalCell <= 40 && _emptyTemplate35Gray != null) { templateGray = _emptyTemplate35Gray; templateArgb = _emptyTemplate35Argb!; templateW = _emptyTemplate35W; templateH = _emptyTemplate35H; templateStride = _emptyTemplate35Stride; } else if (_emptyTemplate70Gray != null) { templateGray = _emptyTemplate70Gray; templateArgb = _emptyTemplate70Argb!; templateW = _emptyTemplate70W; templateH = _emptyTemplate70H; templateStride = _emptyTemplate70Stride; } else { return new ErrorResponse("Empty cell templates not found in assets/"); } // Convert captured bitmap to grayscale + keep ARGB for border color comparison var (captureGray, captureArgb, captureStride) = ImageUtils.BitmapToGrayAndArgb(bitmap); int captureW = bitmap.Width; // Border to skip (outer pixels may differ between cells) int border = Math.Max(2, nominalCell / 10); // Pre-compute template average for the inner region long templateSum = 0; int innerCount = 0; for (int ty = border; ty < templateH - border; ty++) for (int tx = border; tx < templateW - border; tx++) { templateSum += templateGray[ty * templateW + tx]; innerCount++; } // Threshold for mean absolute difference — default 6 double diffThreshold = req.Threshold > 0 ? req.Threshold : 2; bool debug = req.Debug; if (debug) Console.Error.WriteLine($"Grid: {cols}x{rows}, cellW={cellW:F1}, cellH={cellH:F1}, border={border}, threshold={diffThreshold}"); var cells = new List>(); for (int row = 0; row < rows; row++) { var rowList = new List(); var debugDiffs = new List(); for (int col = 0; col < cols; col++) { int cx0 = (int)(col * cellW); int cy0 = (int)(row * cellH); int cw = (int)Math.Min(cellW, captureW - cx0); int ch = (int)Math.Min(cellH, bitmap.Height - cy0); // Compare inner pixels of cell vs template long diffSum = 0; int compared = 0; int innerW = Math.Min(cw, templateW) - border; int innerH = Math.Min(ch, templateH) - border; for (int py = border; py < innerH; py++) { for (int px = border; px < innerW; px++) { int cellVal = captureGray[(cy0 + py) * captureW + (cx0 + px)]; int tmplVal = templateGray[py * templateW + px]; diffSum += Math.Abs(cellVal - tmplVal); compared++; } } double meanDiff = compared > 0 ? (double)diffSum / compared : 0; bool occupied = meanDiff > diffThreshold; rowList.Add(occupied); if (debug) debugDiffs.Add($"{meanDiff,5:F1}{(occupied ? "*" : " ")}"); } cells.Add(rowList); if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}"); } // ── Item detection: compare border pixels to empty template (grayscale) ── // Items have a colored tint behind them that shows through grid lines. // Compare each cell's border strip against the template's border pixels. // If they differ → item tint present → cells belong to same item. int[] parent = new int[rows * cols]; for (int i = 0; i < parent.Length; i++) parent[i] = i; int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; } void Union(int a, int b) { parent[Find(a)] = Find(b); } int stripWidth = Math.Max(2, border / 2); int stripInset = (int)(cellW * 0.15); double borderDiffThresh = 15.0; for (int row = 0; row < rows; row++) { for (int col = 0; col < cols; col++) { if (!cells[row][col]) continue; int cx0 = (int)(col * cellW); int cy0 = (int)(row * cellH); // Check right neighbor if (col + 1 < cols && cells[row][col + 1]) { long diffSum = 0; int cnt = 0; int xStart = (int)((col + 1) * cellW) - stripWidth; int yFrom = cy0 + stripInset; int yTo = (int)((row + 1) * cellH) - stripInset; for (int sy = yFrom; sy < yTo; sy += 2) { int tmplY = sy - cy0; for (int sx = xStart; sx < xStart + stripWidth * 2; sx++) { if (sx < 0 || sx >= captureW) continue; int tmplX = sx - cx0; if (tmplX < 0 || tmplX >= templateW) continue; diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]); cnt++; } } double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0; if (debug) Console.Error.WriteLine($" H ({row},{col})->({row},{col+1}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}"); if (meanDiff > borderDiffThresh) Union(row * cols + col, row * cols + col + 1); } // Check bottom neighbor if (row + 1 < rows && cells[row + 1][col]) { long diffSum = 0; int cnt = 0; int yStart = (int)((row + 1) * cellH) - stripWidth; int xFrom = cx0 + stripInset; int xTo = (int)((col + 1) * cellW) - stripInset; for (int sx = xFrom; sx < xTo; sx += 2) { int tmplX = sx - cx0; for (int sy = yStart; sy < yStart + stripWidth * 2; sy++) { if (sy < 0 || sy >= bitmap.Height) continue; int tmplY = sy - cy0; if (tmplY < 0 || tmplY >= templateH) continue; diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]); cnt++; } } double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0; if (debug) Console.Error.WriteLine($" V ({row},{col})->({row+1},{col}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}"); if (meanDiff > borderDiffThresh) Union(row * cols + col, (row + 1) * cols + col); } } } // Extract items from union-find groups var groups = new Dictionary>(); for (int row = 0; row < rows; row++) for (int col = 0; col < cols; col++) if (cells[row][col]) { int root = Find(row * cols + col); if (!groups.ContainsKey(root)) groups[root] = []; groups[root].Add((row, col)); } var items = new List(); foreach (var group in groups.Values) { int minR = group.Min(c => c.row); int maxR = group.Max(c => c.row); int minC = group.Min(c => c.col); int maxC = group.Max(c => c.col); items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 }); } if (debug) { Console.Error.WriteLine($" Items found: {items.Count}"); foreach (var item in items) Console.Error.WriteLine($" ({item.Row},{item.Col}) {item.W}x{item.H}"); } // ── Visual matching: find cells similar to target ── List? matches = null; if (req.TargetRow >= 0 && req.TargetCol >= 0 && req.TargetRow < rows && req.TargetCol < cols && cells[req.TargetRow][req.TargetCol]) { matches = FindMatchingCells( captureGray, captureW, bitmap.Height, cells, rows, cols, cellW, cellH, border, req.TargetRow, req.TargetCol, debug); } return new GridResponse { Cells = cells, Items = items, Matches = matches }; } /// /// Find all occupied cells visually similar to the target cell using full-resolution NCC. /// private List FindMatchingCells( byte[] gray, int imgW, int imgH, List> cells, int rows, int cols, float cellW, float cellH, int border, int targetRow, int targetCol, bool debug) { int innerW = (int)cellW - border * 2; int innerH = (int)cellH - border * 2; if (innerW <= 4 || innerH <= 4) return []; int tCx0 = (int)(targetCol * cellW) + border; int tCy0 = (int)(targetRow * cellH) + border; int tInnerW = Math.Min(innerW, imgW - tCx0); int tInnerH = Math.Min(innerH, imgH - tCy0); if (tInnerW < innerW || tInnerH < innerH) return []; int n = innerW * innerH; // Pre-compute target cell pixels and stats double[] targetPixels = new double[n]; double tMean = 0; for (int py = 0; py < innerH; py++) for (int px = 0; px < innerW; px++) { double v = gray[(tCy0 + py) * imgW + (tCx0 + px)]; targetPixels[py * innerW + px] = v; tMean += v; } tMean /= n; double tStd = 0; for (int i = 0; i < n; i++) tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean); tStd = Math.Sqrt(tStd / n); if (debug) Console.Error.WriteLine($" Match target ({targetRow},{targetCol}): {innerW}x{innerH} ({n}px), mean={tMean:F1}, std={tStd:F1}"); if (tStd < 3.0) return []; double matchThreshold = 0.70; var matches = new List(); for (int row = 0; row < rows; row++) { for (int col = 0; col < cols; col++) { if (!cells[row][col]) continue; if (row == targetRow && col == targetCol) continue; int cx0 = (int)(col * cellW) + border; int cy0 = (int)(row * cellH) + border; int cInnerW = Math.Min(innerW, imgW - cx0); int cInnerH = Math.Min(innerH, imgH - cy0); if (cInnerW < innerW || cInnerH < innerH) continue; // Compute NCC at full resolution double cMean = 0; for (int py = 0; py < innerH; py++) for (int px = 0; px < innerW; px++) cMean += gray[(cy0 + py) * imgW + (cx0 + px)]; cMean /= n; double cStd = 0, cross = 0; for (int py = 0; py < innerH; py++) for (int px = 0; px < innerW; px++) { double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean; double tv = targetPixels[py * innerW + px] - tMean; cStd += cv * cv; cross += tv * cv; } cStd = Math.Sqrt(cStd / n); double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0; if (debug && ncc > 0.5) Console.Error.WriteLine($" ({row},{col}): NCC={ncc:F3}"); if (ncc >= matchThreshold) matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) }); } } if (debug) Console.Error.WriteLine($" Matches for ({targetRow},{targetCol}): {matches.Count}"); return matches; } private void LoadTemplatesIfNeeded() { if (_emptyTemplate70Gray != null) return; // Look for templates relative to exe directory var exeDir = AppContext.BaseDirectory; // Templates are in assets/ at project root — walk up from bin/Release/net8.0-.../ var projectRoot = Path.GetFullPath(Path.Combine(exeDir, "..", "..", "..", "..", "..")); var t70Path = Path.Combine(projectRoot, "assets", "empty70.png"); var t35Path = Path.Combine(projectRoot, "assets", "empty35.png"); if (File.Exists(t70Path)) { using var bmp = new Bitmap(t70Path); _emptyTemplate70W = bmp.Width; _emptyTemplate70H = bmp.Height; (_emptyTemplate70Gray, _emptyTemplate70Argb, _emptyTemplate70Stride) = ImageUtils.BitmapToGrayAndArgb(bmp); } if (File.Exists(t35Path)) { using var bmp = new Bitmap(t35Path); _emptyTemplate35W = bmp.Width; _emptyTemplate35H = bmp.Height; (_emptyTemplate35Gray, _emptyTemplate35Argb, _emptyTemplate35Stride) = ImageUtils.BitmapToGrayAndArgb(bmp); } } }