finished item finder
This commit is contained in:
parent
1246884be9
commit
930e00c9cc
7 changed files with 450 additions and 37 deletions
|
|
@ -129,10 +129,13 @@ void HandleCapture(Request req)
|
|||
}
|
||||
|
||||
// Pre-loaded empty cell templates (loaded lazily on first grid scan)
|
||||
// Stored as both grayscale (for occupied detection) and ARGB (for item border detection)
|
||||
byte[]? emptyTemplate70Gray = null;
|
||||
int emptyTemplate70W = 0, emptyTemplate70H = 0;
|
||||
byte[]? emptyTemplate70Argb = null;
|
||||
int emptyTemplate70W = 0, emptyTemplate70H = 0, emptyTemplate70Stride = 0;
|
||||
byte[]? emptyTemplate35Gray = null;
|
||||
int emptyTemplate35W = 0, emptyTemplate35H = 0;
|
||||
byte[]? emptyTemplate35Argb = null;
|
||||
int emptyTemplate35W = 0, emptyTemplate35H = 0, emptyTemplate35Stride = 0;
|
||||
|
||||
void LoadTemplatesIfNeeded()
|
||||
{
|
||||
|
|
@ -150,23 +153,23 @@ void LoadTemplatesIfNeeded()
|
|||
using var bmp = new Bitmap(t70Path);
|
||||
emptyTemplate70W = bmp.Width;
|
||||
emptyTemplate70H = bmp.Height;
|
||||
emptyTemplate70Gray = BitmapToGray(bmp);
|
||||
(emptyTemplate70Gray, emptyTemplate70Argb, emptyTemplate70Stride) = BitmapToGrayAndArgb(bmp);
|
||||
}
|
||||
if (System.IO.File.Exists(t35Path))
|
||||
{
|
||||
using var bmp = new Bitmap(t35Path);
|
||||
emptyTemplate35W = bmp.Width;
|
||||
emptyTemplate35H = bmp.Height;
|
||||
emptyTemplate35Gray = BitmapToGray(bmp);
|
||||
(emptyTemplate35Gray, emptyTemplate35Argb, emptyTemplate35Stride) = BitmapToGrayAndArgb(bmp);
|
||||
}
|
||||
}
|
||||
|
||||
byte[] BitmapToGray(Bitmap bmp)
|
||||
(byte[] gray, byte[] argb, int stride) BitmapToGrayAndArgb(Bitmap bmp)
|
||||
{
|
||||
int w = bmp.Width, h = bmp.Height;
|
||||
var data = bmp.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
|
||||
byte[] pixels = new byte[data.Stride * h];
|
||||
Marshal.Copy(data.Scan0, pixels, 0, pixels.Length);
|
||||
byte[] argb = new byte[data.Stride * h];
|
||||
Marshal.Copy(data.Scan0, argb, 0, argb.Length);
|
||||
bmp.UnlockBits(data);
|
||||
int stride = data.Stride;
|
||||
|
||||
|
|
@ -175,8 +178,14 @@ byte[] BitmapToGray(Bitmap bmp)
|
|||
for (int x = 0; x < w; x++)
|
||||
{
|
||||
int i = y * stride + x * 4;
|
||||
gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
|
||||
gray[y * w + x] = (byte)((argb[i] + argb[i + 1] + argb[i + 2]) / 3);
|
||||
}
|
||||
return (gray, argb, stride);
|
||||
}
|
||||
|
||||
byte[] BitmapToGray(Bitmap bmp)
|
||||
{
|
||||
var (gray, _, _) = BitmapToGrayAndArgb(bmp);
|
||||
return gray;
|
||||
}
|
||||
|
||||
|
|
@ -199,18 +208,23 @@ void HandleGrid(Request req)
|
|||
// Pick the right empty template based on cell size
|
||||
int nominalCell = (int)Math.Round(cellW);
|
||||
byte[]? templateGray;
|
||||
int templateW, templateH;
|
||||
byte[]? templateArgb;
|
||||
int templateW, templateH, templateStride;
|
||||
if (nominalCell <= 40 && emptyTemplate35Gray != null)
|
||||
{
|
||||
templateGray = emptyTemplate35Gray;
|
||||
templateArgb = emptyTemplate35Argb!;
|
||||
templateW = emptyTemplate35W;
|
||||
templateH = emptyTemplate35H;
|
||||
templateStride = emptyTemplate35Stride;
|
||||
}
|
||||
else if (emptyTemplate70Gray != null)
|
||||
{
|
||||
templateGray = emptyTemplate70Gray;
|
||||
templateArgb = emptyTemplate70Argb!;
|
||||
templateW = emptyTemplate70W;
|
||||
templateH = emptyTemplate70H;
|
||||
templateStride = emptyTemplate70Stride;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -218,8 +232,8 @@ void HandleGrid(Request req)
|
|||
return;
|
||||
}
|
||||
|
||||
// Convert captured bitmap to grayscale
|
||||
byte[] captureGray = BitmapToGray(bitmap);
|
||||
// Convert captured bitmap to grayscale + keep ARGB for border color comparison
|
||||
var (captureGray, captureArgb, captureStride) = BitmapToGrayAndArgb(bitmap);
|
||||
int captureW = bitmap.Width;
|
||||
|
||||
// Border to skip (outer pixels may differ between cells)
|
||||
|
|
@ -277,7 +291,209 @@ void HandleGrid(Request req)
|
|||
if (debug) Console.Error.WriteLine($" Row {row,2}: {string.Join(" ", debugDiffs)}");
|
||||
}
|
||||
|
||||
WriteResponse(new GridResponse { Cells = cells });
|
||||
// ── Item detection: compare border pixels to empty template (grayscale) ──
|
||||
// Items have a colored tint behind them that shows through grid lines.
|
||||
// Compare each cell's border strip against the template's border pixels.
|
||||
// If they differ → item tint present → cells belong to same item.
|
||||
int[] parent = new int[rows * cols];
|
||||
for (int i = 0; i < parent.Length; i++) parent[i] = i;
|
||||
|
||||
int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; }
|
||||
void Union(int a, int b) { parent[Find(a)] = Find(b); }
|
||||
|
||||
int stripWidth = Math.Max(2, border / 2);
|
||||
int stripInset = (int)(cellW * 0.15);
|
||||
double borderDiffThresh = 15.0;
|
||||
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
if (!cells[row][col]) continue;
|
||||
int cx0 = (int)(col * cellW);
|
||||
int cy0 = (int)(row * cellH);
|
||||
|
||||
// Check right neighbor
|
||||
if (col + 1 < cols && cells[row][col + 1])
|
||||
{
|
||||
long diffSum = 0; int cnt = 0;
|
||||
int xStart = (int)((col + 1) * cellW) - stripWidth;
|
||||
int yFrom = cy0 + stripInset;
|
||||
int yTo = (int)((row + 1) * cellH) - stripInset;
|
||||
for (int sy = yFrom; sy < yTo; sy += 2)
|
||||
{
|
||||
int tmplY = sy - cy0;
|
||||
for (int sx = xStart; sx < xStart + stripWidth * 2; sx++)
|
||||
{
|
||||
if (sx < 0 || sx >= captureW) continue;
|
||||
int tmplX = sx - cx0;
|
||||
if (tmplX < 0 || tmplX >= templateW) continue;
|
||||
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
||||
if (debug) Console.Error.WriteLine($" H ({row},{col})->({row},{col+1}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
||||
if (meanDiff > borderDiffThresh)
|
||||
Union(row * cols + col, row * cols + col + 1);
|
||||
}
|
||||
|
||||
// Check bottom neighbor
|
||||
if (row + 1 < rows && cells[row + 1][col])
|
||||
{
|
||||
long diffSum = 0; int cnt = 0;
|
||||
int yStart = (int)((row + 1) * cellH) - stripWidth;
|
||||
int xFrom = cx0 + stripInset;
|
||||
int xTo = (int)((col + 1) * cellW) - stripInset;
|
||||
for (int sx = xFrom; sx < xTo; sx += 2)
|
||||
{
|
||||
int tmplX = sx - cx0;
|
||||
for (int sy = yStart; sy < yStart + stripWidth * 2; sy++)
|
||||
{
|
||||
if (sy < 0 || sy >= bitmap.Height) continue;
|
||||
int tmplY = sy - cy0;
|
||||
if (tmplY < 0 || tmplY >= templateH) continue;
|
||||
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
|
||||
cnt++;
|
||||
}
|
||||
}
|
||||
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
|
||||
if (debug) Console.Error.WriteLine($" V ({row},{col})->({row+1},{col}): {meanDiff:F1}{(meanDiff > borderDiffThresh ? " SAME" : "")}");
|
||||
if (meanDiff > borderDiffThresh)
|
||||
Union(row * cols + col, (row + 1) * cols + col);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract items from union-find groups
|
||||
var groups = new Dictionary<int, List<(int row, int col)>>();
|
||||
for (int row = 0; row < rows; row++)
|
||||
for (int col = 0; col < cols; col++)
|
||||
if (cells[row][col])
|
||||
{
|
||||
int root = Find(row * cols + col);
|
||||
if (!groups.ContainsKey(root)) groups[root] = [];
|
||||
groups[root].Add((row, col));
|
||||
}
|
||||
|
||||
var items = new List<GridItem>();
|
||||
foreach (var group in groups.Values)
|
||||
{
|
||||
int minR = group.Min(c => c.row);
|
||||
int maxR = group.Max(c => c.row);
|
||||
int minC = group.Min(c => c.col);
|
||||
int maxC = group.Max(c => c.col);
|
||||
items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 });
|
||||
}
|
||||
|
||||
if (debug)
|
||||
{
|
||||
Console.Error.WriteLine($" Items found: {items.Count}");
|
||||
foreach (var item in items)
|
||||
Console.Error.WriteLine($" ({item.Row},{item.Col}) {item.W}x{item.H}");
|
||||
}
|
||||
|
||||
// ── Visual matching: find cells similar to target ──
|
||||
List<GridMatch>? matches = null;
|
||||
if (req.TargetRow >= 0 && req.TargetCol >= 0 &&
|
||||
req.TargetRow < rows && req.TargetCol < cols &&
|
||||
cells[req.TargetRow][req.TargetCol])
|
||||
{
|
||||
matches = FindMatchingCells(
|
||||
captureGray, captureW, bitmap.Height,
|
||||
cells, rows, cols, cellW, cellH, border,
|
||||
req.TargetRow, req.TargetCol, debug);
|
||||
}
|
||||
|
||||
WriteResponse(new GridResponse { Cells = cells, Items = items, Matches = matches });
|
||||
}
|
||||
|
||||
/// Find all occupied cells visually similar to the target cell using full-resolution NCC.
|
||||
/// Full resolution gives better discrimination — sockets are a small fraction of total pixels.
|
||||
List<GridMatch> FindMatchingCells(
|
||||
byte[] gray, int imgW, int imgH,
|
||||
List<List<bool>> cells, int rows, int cols,
|
||||
float cellW, float cellH, int border,
|
||||
int targetRow, int targetCol, bool debug)
|
||||
{
|
||||
int innerW = (int)cellW - border * 2;
|
||||
int innerH = (int)cellH - border * 2;
|
||||
if (innerW <= 4 || innerH <= 4) return [];
|
||||
|
||||
int tCx0 = (int)(targetCol * cellW) + border;
|
||||
int tCy0 = (int)(targetRow * cellH) + border;
|
||||
int tInnerW = Math.Min(innerW, imgW - tCx0);
|
||||
int tInnerH = Math.Min(innerH, imgH - tCy0);
|
||||
if (tInnerW < innerW || tInnerH < innerH) return [];
|
||||
|
||||
int n = innerW * innerH;
|
||||
|
||||
// Pre-compute target cell pixels and stats
|
||||
double[] targetPixels = new double[n];
|
||||
double tMean = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
{
|
||||
double v = gray[(tCy0 + py) * imgW + (tCx0 + px)];
|
||||
targetPixels[py * innerW + px] = v;
|
||||
tMean += v;
|
||||
}
|
||||
tMean /= n;
|
||||
|
||||
double tStd = 0;
|
||||
for (int i = 0; i < n; i++)
|
||||
tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean);
|
||||
tStd = Math.Sqrt(tStd / n);
|
||||
|
||||
if (debug) Console.Error.WriteLine($" Match target ({targetRow},{targetCol}): {innerW}x{innerH} ({n}px), mean={tMean:F1}, std={tStd:F1}");
|
||||
if (tStd < 3.0) return [];
|
||||
|
||||
double matchThreshold = 0.70;
|
||||
var matches = new List<GridMatch>();
|
||||
|
||||
for (int row = 0; row < rows; row++)
|
||||
{
|
||||
for (int col = 0; col < cols; col++)
|
||||
{
|
||||
if (!cells[row][col]) continue;
|
||||
if (row == targetRow && col == targetCol) continue;
|
||||
|
||||
int cx0 = (int)(col * cellW) + border;
|
||||
int cy0 = (int)(row * cellH) + border;
|
||||
int cInnerW = Math.Min(innerW, imgW - cx0);
|
||||
int cInnerH = Math.Min(innerH, imgH - cy0);
|
||||
if (cInnerW < innerW || cInnerH < innerH) continue;
|
||||
|
||||
// Compute NCC at full resolution
|
||||
double cMean = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
cMean += gray[(cy0 + py) * imgW + (cx0 + px)];
|
||||
cMean /= n;
|
||||
|
||||
double cStd = 0, cross = 0;
|
||||
for (int py = 0; py < innerH; py++)
|
||||
for (int px = 0; px < innerW; px++)
|
||||
{
|
||||
double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean;
|
||||
double tv = targetPixels[py * innerW + px] - tMean;
|
||||
cStd += cv * cv;
|
||||
cross += tv * cv;
|
||||
}
|
||||
cStd = Math.Sqrt(cStd / n);
|
||||
|
||||
double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0;
|
||||
|
||||
if (debug && ncc > 0.5)
|
||||
Console.Error.WriteLine($" ({row},{col}): NCC={ncc:F3}");
|
||||
|
||||
if (ncc >= matchThreshold)
|
||||
matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) });
|
||||
}
|
||||
}
|
||||
|
||||
if (debug) Console.Error.WriteLine($" Matches for ({targetRow},{targetCol}): {matches.Count}");
|
||||
return matches;
|
||||
}
|
||||
|
||||
void HandleDetectGrid(Request req)
|
||||
|
|
@ -788,6 +1004,12 @@ class Request
|
|||
|
||||
[JsonPropertyName("debug")]
|
||||
public bool Debug { get; set; }
|
||||
|
||||
[JsonPropertyName("targetRow")]
|
||||
public int TargetRow { get; set; } = -1;
|
||||
|
||||
[JsonPropertyName("targetCol")]
|
||||
public int TargetCol { get; set; } = -1;
|
||||
}
|
||||
|
||||
class RegionRect
|
||||
|
|
@ -884,6 +1106,39 @@ class GridResponse
|
|||
|
||||
[JsonPropertyName("cells")]
|
||||
public List<List<bool>> Cells { get; set; } = [];
|
||||
|
||||
[JsonPropertyName("items")]
|
||||
public List<GridItem>? Items { get; set; }
|
||||
|
||||
[JsonPropertyName("matches")]
|
||||
public List<GridMatch>? Matches { get; set; }
|
||||
}
|
||||
|
||||
class GridItem
|
||||
{
|
||||
[JsonPropertyName("row")]
|
||||
public int Row { get; set; }
|
||||
|
||||
[JsonPropertyName("col")]
|
||||
public int Col { get; set; }
|
||||
|
||||
[JsonPropertyName("w")]
|
||||
public int W { get; set; }
|
||||
|
||||
[JsonPropertyName("h")]
|
||||
public int H { get; set; }
|
||||
}
|
||||
|
||||
class GridMatch
|
||||
{
|
||||
[JsonPropertyName("row")]
|
||||
public int Row { get; set; }
|
||||
|
||||
[JsonPropertyName("col")]
|
||||
public int Col { get; set; }
|
||||
|
||||
[JsonPropertyName("similarity")]
|
||||
public double Similarity { get; set; }
|
||||
}
|
||||
|
||||
class DetectGridResponse
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue