boki/poe2-bot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

switched to new way

Browse source
This commit is contained in:
Boki 2026-02-13 01:12:51 -05:00
parent f22d182c8f
commit 4a65c8e17b
96 changed files with 4991 additions and 10025 deletions
Download patch file Download diff file Expand all files Collapse all files

171
src/Poe2Trade.Screen/DaemonTypes.cs Normal file
View file

@ -0,0 +1,171 @@
using System.Text.Json.Serialization;
namespace Poe2Trade.Screen;
public class OcrWord
{
[JsonPropertyName("text")] public string Text { get; set; } = "";
[JsonPropertyName("x")] public int X { get; set; }
[JsonPropertyName("y")] public int Y { get; set; }
[JsonPropertyName("width")] public int Width { get; set; }
[JsonPropertyName("height")] public int Height { get; set; }
}
public class OcrLine
{
[JsonPropertyName("text")] public string Text { get; set; } = "";
[JsonPropertyName("words")] public List<OcrWord> Words { get; set; } = [];
}
public class OcrResponse
{
public string Text { get; set; } = "";
public List<OcrLine> Lines { get; set; } = [];
}
public class GridItem
{
[JsonPropertyName("row")] public int Row { get; set; }
[JsonPropertyName("col")] public int Col { get; set; }
[JsonPropertyName("w")] public int W { get; set; }
[JsonPropertyName("h")] public int H { get; set; }
}
public class GridMatch
{
[JsonPropertyName("row")] public int Row { get; set; }
[JsonPropertyName("col")] public int Col { get; set; }
[JsonPropertyName("similarity")] public double Similarity { get; set; }
}
public class GridScanResult
{
public bool[][] Cells { get; set; } = [];
public List<GridItem> Items { get; set; } = [];
public List<GridMatch>? Matches { get; set; }
}
public class DiffOcrResponse
{
public string Text { get; set; } = "";
public List<OcrLine> Lines { get; set; } = [];
public Poe2Trade.Core.Region? Region { get; set; }
}
public class TemplateMatchResult
{
public int X { get; set; }
public int Y { get; set; }
public int Width { get; set; }
public int Height { get; set; }
public double Confidence { get; set; }
}
// -- Parameter types --
public sealed class DiffCropParams
{
[JsonPropertyName("diffThresh")]
public int DiffThresh { get; set; } = 20;
[JsonPropertyName("rowThreshDiv")]
public int RowThreshDiv { get; set; } = 40;
[JsonPropertyName("colThreshDiv")]
public int ColThreshDiv { get; set; } = 8;
[JsonPropertyName("maxGap")]
public int MaxGap { get; set; } = 20;
[JsonPropertyName("trimCutoff")]
public double TrimCutoff { get; set; } = 0.4;
[JsonPropertyName("ocrPad")]
public int OcrPad { get; set; } = 10;
}
public sealed class OcrParams
{
// preprocessing
[JsonPropertyName("kernelSize")]
public int KernelSize { get; set; } = 41;
[JsonPropertyName("upscale")]
public int Upscale { get; set; } = 2;
[JsonPropertyName("useBackgroundSub")]
public bool UseBackgroundSub { get; set; } = true;
[JsonPropertyName("dimPercentile")]
public int DimPercentile { get; set; } = 40;
[JsonPropertyName("textThresh")]
public int TextThresh { get; set; } = 60;
[JsonPropertyName("softThreshold")]
public bool SoftThreshold { get; set; } = false;
// EasyOCR tuning
[JsonPropertyName("mergeGap")]
public int MergeGap { get; set; } = 0;
[JsonPropertyName("linkThreshold")]
public double? LinkThreshold { get; set; }
[JsonPropertyName("textThreshold")]
public double? TextThreshold { get; set; }
[JsonPropertyName("lowText")]
public double? LowText { get; set; }
[JsonPropertyName("widthThs")]
public double? WidthThs { get; set; }
[JsonPropertyName("paragraph")]
public bool? Paragraph { get; set; }
}
public sealed class DiffOcrParams
{
[JsonPropertyName("crop")]
public DiffCropParams Crop { get; set; } = new();
[JsonPropertyName("ocr")]
public OcrParams Ocr { get; set; } = new();
}
public sealed class EdgeCropParams
{
[JsonPropertyName("darkThresh")]
public int DarkThresh { get; set; } = 40;
[JsonPropertyName("minDarkRun")]
public int MinDarkRun { get; set; } = 200;
[JsonPropertyName("runGapTolerance")]
public int RunGapTolerance { get; set; } = 15;
[JsonPropertyName("rowThreshDiv")]
public int RowThreshDiv { get; set; } = 40;
[JsonPropertyName("colThreshDiv")]
public int ColThreshDiv { get; set; } = 8;
[JsonPropertyName("maxGap")]
public int MaxGap { get; set; } = 15;
[JsonPropertyName("trimCutoff")]
public double TrimCutoff { get; set; } = 0.3;
[JsonPropertyName("ocrPad")]
public int OcrPad { get; set; } = 10;
}
public sealed class EdgeOcrParams
{
[JsonPropertyName("crop")]
public EdgeCropParams Crop { get; set; } = new();
[JsonPropertyName("ocr")]
public OcrParams Ocr { get; set; } = new();
}

157
src/Poe2Trade.Screen/DetectGridHandler.cs Normal file
View file

@ -0,0 +1,157 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Serilog;
using Region = Poe2Trade.Core.Region;
class DetectGridHandler
{
public DetectGridResult Detect(Region region, int minCellSize = 20, int maxCellSize = 70,
string? file = null, bool debug = false)
{
Bitmap bitmap = ScreenCapture.CaptureOrLoad(file, region);
int w = bitmap.Width;
int h = bitmap.Height;
var bmpData = bitmap.LockBits(
new Rectangle(0, 0, w, h),
ImageLockMode.ReadOnly,
PixelFormat.Format32bppArgb
);
byte[] pixels = new byte[bmpData.Stride * h];
Marshal.Copy(bmpData.Scan0, pixels, 0, pixels.Length);
bitmap.UnlockBits(bmpData);
int stride = bmpData.Stride;
byte[] gray = new byte[w * h];
for (int y = 0; y < h; y++)
for (int x = 0; x < w; x++)
{
int i = y * stride + x * 4;
gray[y * w + x] = (byte)((pixels[i] + pixels[i + 1] + pixels[i + 2]) / 3);
}
bitmap.Dispose();
int bandH = 200;
int bandStep = 40;
const int veryDarkPixelThresh = 12;
const double gridSegThresh = 0.25;
var candidates = new List<(int bandY, int cellW, double hAc, int hLeft, int hRight)>();
for (int by = 0; by + bandH <= h; by += bandStep)
{
double[] darkDensity = new double[w];
for (int x = 0; x < w; x++)
{
int count = 0;
for (int y = by; y < by + bandH; y++)
{
if (gray[y * w + x] < veryDarkPixelThresh) count++;
}
darkDensity[x] = (double)count / bandH;
}
var gridSegs = SignalProcessing.FindDarkDensitySegments(darkDensity, gridSegThresh, 200);
foreach (var (segLeft, segRight) in gridSegs)
{
int segLen = segRight - segLeft;
double[] segment = new double[segLen];
Array.Copy(darkDensity, segLeft, segment, 0, segLen);
var (period, acScore) = SignalProcessing.FindPeriodWithScore(segment, minCellSize, maxCellSize);
if (period <= 0) continue;
var (extLeft, extRight) = SignalProcessing.FindGridExtent(segment, period);
if (extLeft < 0) continue;
int absLeft = segLeft + extLeft;
int absRight = segLeft + extRight;
int extent = absRight - absLeft;
if (extent < period * 8 || extent < 200) continue;
candidates.Add((by, period, acScore, absLeft, absRight));
}
}
candidates.Sort((a, b) =>
{
double sa = a.hAc * (a.hRight - a.hLeft);
double sb = b.hAc * (b.hRight - b.hLeft);
return sb.CompareTo(sa);
});
// Pass 2: Verify vertical periodicity
foreach (var cand in candidates.Take(10))
{
int colSpan = cand.hRight - cand.hLeft;
if (colSpan < cand.cellW * 3) continue;
double[] rowDensity = new double[h];
for (int y = 0; y < h; y++)
{
int count = 0;
for (int x = cand.hLeft; x < cand.hRight; x++)
{
if (gray[y * w + x] < veryDarkPixelThresh) count++;
}
rowDensity[y] = (double)count / colSpan;
}
var vGridSegs = SignalProcessing.FindDarkDensitySegments(rowDensity, gridSegThresh, 100);
if (vGridSegs.Count == 0) continue;
var (vSegTop, vSegBottom) = vGridSegs.OrderByDescending(s => s.end - s.start).First();
int vSegLen = vSegBottom - vSegTop;
double[] vSegment = new double[vSegLen];
Array.Copy(rowDensity, vSegTop, vSegment, 0, vSegLen);
var (cellH, vAc) = SignalProcessing.FindPeriodWithScore(vSegment, minCellSize, maxCellSize);
if (cellH <= 0) continue;
var (extTop, extBottom) = SignalProcessing.FindGridExtent(vSegment, cellH);
if (extTop < 0) continue;
int top = vSegTop + extTop;
int bottom = vSegTop + extBottom;
int vExtent = bottom - top;
if (vExtent < cellH * 3 || vExtent < 100) continue;
int gridW = cand.hRight - cand.hLeft;
int gridH = bottom - top;
int cols = Math.Max(2, (int)Math.Round((double)gridW / cand.cellW));
int rows = Math.Max(2, (int)Math.Round((double)gridH / cellH));
gridW = cols * cand.cellW;
gridH = rows * cellH;
return new DetectGridResult
{
Detected = true,
Region = new Region(region.X + cand.hLeft, region.Y + top, gridW, gridH),
Cols = cols,
Rows = rows,
CellWidth = Math.Round((double)gridW / cols, 1),
CellHeight = Math.Round((double)gridH / rows, 1),
};
}
return new DetectGridResult { Detected = false };
}
}
public class DetectGridResult
{
public bool Detected { get; set; }
public Region? Region { get; set; }
public int Cols { get; set; }
public int Rows { get; set; }
public double CellWidth { get; set; }
public double CellHeight { get; set; }
}

368
src/Poe2Trade.Screen/DiffCropHandler.cs Normal file
View file

@ -0,0 +1,368 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Serilog;
using Region = Poe2Trade.Core.Region;
class DiffCropHandler
{
private Bitmap? _referenceFrame;
private Region? _referenceRegion;
public void HandleSnapshot(string? file = null, Region? region = null)
{
_referenceFrame?.Dispose();
_referenceFrame = ScreenCapture.CaptureOrLoad(file, region);
_referenceRegion = region;
}
public void HandleScreenshot(string path, Region? region = null)
{
var bitmap = _referenceFrame ?? ScreenCapture.CaptureOrLoad(null, region);
var format = ImageUtils.GetImageFormat(path);
var dir = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
Directory.CreateDirectory(dir);
bitmap.Save(path, format);
if (bitmap != _referenceFrame) bitmap.Dispose();
}
public byte[] HandleCapture(Region? region = null)
{
using var bitmap = ScreenCapture.CaptureOrLoad(null, region);
using var ms = new MemoryStream();
bitmap.Save(ms, System.Drawing.Imaging.ImageFormat.Png);
return ms.ToArray();
}
/// <summary>
/// Diff detection + crop only. Returns the raw tooltip crop bitmap and region,
/// or null if no tooltip detected. Caller is responsible for disposing the bitmaps.
/// </summary>
public (Bitmap cropped, Bitmap refCropped, Bitmap current, Region region)? DiffCrop(
DiffCropParams c, string? file = null, Region? region = null)
{
if (_referenceFrame == null)
return null;
var diffRegion = region ?? _referenceRegion;
int baseX = diffRegion?.X ?? 0;
int baseY = diffRegion?.Y ?? 0;
var current = ScreenCapture.CaptureOrLoad(file, diffRegion);
Bitmap refForDiff = _referenceFrame;
bool disposeRef = false;
if (diffRegion != null)
{
if (_referenceRegion == null)
{
var croppedRef = CropBitmap(_referenceFrame, diffRegion);
if (croppedRef == null)
{
current.Dispose();
return null;
}
refForDiff = croppedRef;
disposeRef = true;
}
else if (!RegionsEqual(diffRegion, _referenceRegion))
{
int offX = diffRegion.X - _referenceRegion.X;
int offY = diffRegion.Y - _referenceRegion.Y;
if (offX < 0 || offY < 0 || offX + diffRegion.Width > _referenceFrame.Width || offY + diffRegion.Height > _referenceFrame.Height)
{
current.Dispose();
return null;
}
var croppedRef = CropBitmap(_referenceFrame, new Region(offX, offY, diffRegion.Width, diffRegion.Height));
if (croppedRef == null)
{
current.Dispose();
return null;
}
refForDiff = croppedRef;
disposeRef = true;
}
}
int w = Math.Min(refForDiff.Width, current.Width);
int h = Math.Min(refForDiff.Height, current.Height);
var refData = refForDiff.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
byte[] refPx = new byte[refData.Stride * h];
Marshal.Copy(refData.Scan0, refPx, 0, refPx.Length);
refForDiff.UnlockBits(refData);
int stride = refData.Stride;
var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
byte[] curPx = new byte[curData.Stride * h];
Marshal.Copy(curData.Scan0, curPx, 0, curPx.Length);
current.UnlockBits(curData);
int diffThresh = c.DiffThresh;
// Pass 1: parallel row diff
int[] rowCounts = new int[h];
Parallel.For(0, h, y =>
{
int count = 0;
int rowOffset = y * stride;
for (int x = 0; x < w; x++)
{
int i = rowOffset + x * 4;
int darker = (refPx[i] - curPx[i]) + (refPx[i + 1] - curPx[i + 1]) + (refPx[i + 2] - curPx[i + 2]);
if (darker > diffThresh)
count++;
}
rowCounts[y] = count;
});
int totalChanged = 0;
for (int y = 0; y < h; y++) totalChanged += rowCounts[y];
if (totalChanged == 0)
{
current.Dispose();
if (disposeRef) refForDiff.Dispose();
return null;
}
int maxGap = c.MaxGap;
int rowThresh = w / c.RowThreshDiv;
int bestRowStart = 0, bestRowEnd = 0, bestRowLen = 0;
int curRowStart = -1, lastActiveRow = -1;
for (int y = 0; y < h; y++)
{
if (rowCounts[y] >= rowThresh)
{
if (curRowStart < 0) curRowStart = y;
lastActiveRow = y;
}
else if (curRowStart >= 0 && y - lastActiveRow > maxGap)
{
int len = lastActiveRow - curRowStart + 1;
if (len > bestRowLen) { bestRowStart = curRowStart; bestRowEnd = lastActiveRow; bestRowLen = len; }
curRowStart = -1;
}
}
if (curRowStart >= 0)
{
int len = lastActiveRow - curRowStart + 1;
if (len > bestRowLen) { bestRowStart = curRowStart; bestRowEnd = lastActiveRow; bestRowLen = len; }
}
// Pass 2: parallel column diff
int[] colCounts = new int[w];
int rowRangeLen = bestRowEnd - bestRowStart + 1;
if (rowRangeLen <= 200)
{
for (int y = bestRowStart; y <= bestRowEnd; y++)
{
int rowOffset = y * stride;
for (int x = 0; x < w; x++)
{
int i = rowOffset + x * 4;
int darker = (refPx[i] - curPx[i]) + (refPx[i + 1] - curPx[i + 1]) + (refPx[i + 2] - curPx[i + 2]);
if (darker > diffThresh)
colCounts[x]++;
}
}
}
else
{
Parallel.For(bestRowStart, bestRowEnd + 1,
() => new int[w],
(y, _, localCols) =>
{
int rowOffset = y * stride;
for (int x = 0; x < w; x++)
{
int i = rowOffset + x * 4;
int darker = (refPx[i] - curPx[i]) + (refPx[i + 1] - curPx[i + 1]) + (refPx[i + 2] - curPx[i + 2]);
if (darker > diffThresh)
localCols[x]++;
}
return localCols;
},
localCols =>
{
for (int x = 0; x < w; x++)
Interlocked.Add(ref colCounts[x], localCols[x]);
});
}
int tooltipHeight = bestRowEnd - bestRowStart + 1;
int colThresh = tooltipHeight / c.ColThreshDiv;
int bestColStart = 0, bestColEnd = 0, bestColLen = 0;
int curColStart = -1, lastActiveCol = -1;
for (int x = 0; x < w; x++)
{
if (colCounts[x] >= colThresh)
{
if (curColStart < 0) curColStart = x;
lastActiveCol = x;
}
else if (curColStart >= 0 && x - lastActiveCol > maxGap)
{
int len = lastActiveCol - curColStart + 1;
if (len > bestColLen) { bestColStart = curColStart; bestColEnd = lastActiveCol; bestColLen = len; }
curColStart = -1;
}
}
if (curColStart >= 0)
{
int len = lastActiveCol - curColStart + 1;
if (len > bestColLen) { bestColStart = curColStart; bestColEnd = lastActiveCol; bestColLen = len; }
}
Log.Debug("diff-crop: changed={Changed} rows={RowStart}-{RowEnd}({RowLen}) cols={ColStart}-{ColEnd}({ColLen})",
totalChanged, bestRowStart, bestRowEnd, bestRowLen, bestColStart, bestColEnd, bestColLen);
if (bestRowLen < 50 || bestColLen < 50)
{
Log.Debug("diff-crop: no tooltip-sized region found");
current.Dispose();
if (disposeRef) refForDiff.Dispose();
return null;
}
int minX = bestColStart;
int minY = bestRowStart;
int maxX = Math.Min(bestColEnd, w - 1);
int maxY = Math.Min(bestRowEnd, h - 1);
// Boundary extension
int extRowThresh = Math.Max(1, rowThresh / 4);
int extColThresh = Math.Max(1, colThresh / 4);
int extTop = Math.Max(0, minY - maxGap);
for (int y = minY - 1; y >= extTop; y--)
{
if (rowCounts[y] >= extRowThresh) minY = y;
else break;
}
int extBottom = Math.Min(h - 1, maxY + maxGap);
for (int y = maxY + 1; y <= extBottom; y++)
{
if (rowCounts[y] >= extRowThresh) maxY = y;
else break;
}
int extLeft = Math.Max(0, minX - maxGap);
for (int x = minX - 1; x >= extLeft; x--)
{
if (colCounts[x] >= extColThresh) minX = x;
else break;
}
int extRight = Math.Min(w - 1, maxX + maxGap);
for (int x = maxX + 1; x <= extRight; x++)
{
if (colCounts[x] >= extColThresh) maxX = x;
else break;
}
// Trim low-density edges
int colSpan = maxX - minX + 1;
if (colSpan > 50)
{
int q1 = minX + colSpan / 4;
int q3 = minX + colSpan * 3 / 4;
long midSum = 0;
int midCount = 0;
for (int x = q1; x <= q3; x++) { midSum += colCounts[x]; midCount++; }
double avgMidDensity = (double)midSum / Math.Max(1, midCount);
double cutoff = avgMidDensity * c.TrimCutoff;
while (minX < maxX - 50 && colCounts[minX] < cutoff)
minX++;
while (maxX > minX + 50 && colCounts[maxX] < cutoff)
maxX--;
}
int rowSpan = maxY - minY + 1;
if (rowSpan > 50)
{
int q1 = minY + rowSpan / 4;
int q3 = minY + rowSpan * 3 / 4;
long midSum = 0;
int midCount = 0;
for (int y = q1; y <= q3; y++) { midSum += rowCounts[y]; midCount++; }
double avgMidDensity = (double)midSum / Math.Max(1, midCount);
double cutoff = avgMidDensity * c.TrimCutoff;
while (minY < maxY - 50 && rowCounts[minY] < cutoff)
minY++;
while (maxY > minY + 50 && rowCounts[maxY] < cutoff)
maxY--;
}
int rw = maxX - minX + 1;
int rh = maxY - minY + 1;
var cropped = CropFromBytes(curPx, stride, minX, minY, rw, rh);
var refCropped = CropFromBytes(refPx, stride, minX, minY, rw, rh);
var resultRegion = new Region(baseX + minX, baseY + minY, rw, rh);
Log.Debug("diff-crop: tooltip region ({X},{Y}) {W}x{H}", minX, minY, rw, rh);
if (disposeRef) refForDiff.Dispose();
return (cropped, refCropped, current, resultRegion);
}
private static bool RegionsEqual(Region a, Region b) =>
a.X == b.X && a.Y == b.Y && a.Width == b.Width && a.Height == b.Height;
private static Bitmap? CropBitmap(Bitmap src, Region region)
{
int cx = Math.Max(0, region.X);
int cy = Math.Max(0, region.Y);
int cw = Math.Min(region.Width, src.Width - cx);
int ch = Math.Min(region.Height, src.Height - cy);
if (cw <= 0 || ch <= 0)
return null;
return src.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb);
}
/// <summary>
/// Fast crop from raw pixel bytes.
/// </summary>
private static Bitmap CropFromBytes(byte[] px, int srcStride, int cropX, int cropY, int cropW, int cropH)
{
var bmp = new Bitmap(cropW, cropH, PixelFormat.Format32bppArgb);
var data = bmp.LockBits(new Rectangle(0, 0, cropW, cropH), ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb);
int dstStride = data.Stride;
int rowBytes = cropW * 4;
for (int y = 0; y < cropH; y++)
{
int srcOffset = (cropY + y) * srcStride + cropX * 4;
Marshal.Copy(px, srcOffset, data.Scan0 + y * dstStride, rowBytes);
}
bmp.UnlockBits(data);
return bmp;
}
public static double LevenshteinSimilarity(string a, string b)
{
a = a.ToLowerInvariant();
b = b.ToLowerInvariant();
if (a == b) return 1.0;
int la = a.Length, lb = b.Length;
if (la == 0 || lb == 0) return 0.0;
var d = new int[la + 1, lb + 1];
for (int i = 0; i <= la; i++) d[i, 0] = i;
for (int j = 0; j <= lb; j++) d[0, j] = j;
for (int i = 1; i <= la; i++)
for (int j = 1; j <= lb; j++)
{
int cost = a[i - 1] == b[j - 1] ? 0 : 1;
d[i, j] = Math.Min(Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), d[i - 1, j - 1] + cost);
}
return 1.0 - (double)d[la, lb] / Math.Max(la, lb);
}
}

243
src/Poe2Trade.Screen/EdgeCropHandler.cs Normal file
View file

@ -0,0 +1,243 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Serilog;
using Region = Poe2Trade.Core.Region;
class EdgeCropHandler
{
[StructLayout(LayoutKind.Sequential)]
private struct POINT { public int X, Y; }
[DllImport("user32.dll")]
private static extern bool GetCursorPos(out POINT lpPoint);
public (Bitmap cropped, Bitmap fullCapture, Region region)? EdgeCrop(
EdgeCropParams p, int? cursorX = null, int? cursorY = null, string? file = null)
{
int cx, cy;
if (cursorX.HasValue && cursorY.HasValue)
{
cx = cursorX.Value;
cy = cursorY.Value;
}
else
{
GetCursorPos(out var pt);
cx = pt.X;
cy = pt.Y;
}
var fullCapture = ScreenCapture.CaptureOrLoad(file, null);
int w = fullCapture.Width;
int h = fullCapture.Height;
cx = Math.Clamp(cx, 0, w - 1);
cy = Math.Clamp(cy, 0, h - 1);
var bmpData = fullCapture.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
byte[] px = new byte[bmpData.Stride * h];
Marshal.Copy(bmpData.Scan0, px, 0, px.Length);
fullCapture.UnlockBits(bmpData);
int stride = bmpData.Stride;
int darkThresh = p.DarkThresh;
int colGap = p.RunGapTolerance;
int maxGap = p.MaxGap;
// Phase 1: Per-row horizontal extent
int bandHalf = p.MinDarkRun;
int bandTop = Math.Max(0, cy - bandHalf);
int bandBot = Math.Min(h - 1, cy + bandHalf);
var leftExtents = new List<int>();
var rightExtents = new List<int>();
for (int y = bandTop; y <= bandBot; y++)
{
int rowOff = y * stride;
int seedX = FindDarkSeedInRow(px, stride, w, rowOff, cx, darkThresh, seedRadius: 6);
if (seedX < 0) continue;
int leftEdge = cx;
int gap = 0;
bool foundLeft = false;
int initialBridge = Math.Max(colGap * 4, 12);
for (int x = cx; x >= 0; x--)
{
int i = rowOff + x * 4;
int brightness = (px[i] + px[i + 1] + px[i + 2]) / 3;
if (brightness < darkThresh) { leftEdge = x; gap = 0; foundLeft = true; }
else if (++gap > (foundLeft ? colGap : initialBridge)) break;
}
int rightEdge = cx;
gap = 0;
bool foundRight = false;
for (int x = cx; x < w; x++)
{
int i = rowOff + x * 4;
int brightness = (px[i] + px[i + 1] + px[i + 2]) / 3;
if (brightness < darkThresh) { rightEdge = x; gap = 0; foundRight = true; }
else if (++gap > (foundRight ? colGap : initialBridge)) break;
}
leftExtents.Add(leftEdge);
rightExtents.Add(rightEdge);
}
if (leftExtents.Count < 10)
{
Log.Debug("edge-crop: too few dark rows ({Count})", leftExtents.Count);
fullCapture.Dispose();
return null;
}
leftExtents.Sort();
rightExtents.Sort();
int leftPctIdx = leftExtents.Count / p.RowThreshDiv;
int rightPctIdx = rightExtents.Count * (p.ColThreshDiv - 1) / p.ColThreshDiv;
leftPctIdx = Math.Clamp(leftPctIdx, 0, leftExtents.Count - 1);
rightPctIdx = Math.Clamp(rightPctIdx, 0, rightExtents.Count - 1);
int bestColStart = leftExtents[leftPctIdx];
int bestColEnd = rightExtents[rightPctIdx];
if (bestColEnd - bestColStart + 1 < 50)
{
Log.Debug("edge-crop: horizontal extent too small");
fullCapture.Dispose();
return null;
}
// Phase 2: Per-column vertical extent
int colBandHalf = (bestColEnd - bestColStart + 1) / 3;
int colBandLeft = Math.Max(bestColStart, cx - colBandHalf);
int colBandRight = Math.Min(bestColEnd, cx + colBandHalf);
var topExtents = new List<int>();
var bottomExtents = new List<int>();
int maxGapUp = maxGap * 3;
for (int x = colBandLeft; x <= colBandRight; x++)
{
int seedY = FindDarkSeedInColumn(px, stride, h, x, cy, darkThresh, seedRadius: 6);
if (seedY < 0) continue;
int topEdge = cy;
int gap = 0;
bool foundTop = false;
int initialBridgeUp = Math.Max(maxGapUp * 2, 12);
for (int y = cy; y >= 0; y--)
{
int i = y * stride + x * 4;
int brightness = (px[i] + px[i + 1] + px[i + 2]) / 3;
if (brightness < darkThresh) { topEdge = y; gap = 0; foundTop = true; }
else if (++gap > (foundTop ? maxGapUp : initialBridgeUp)) break;
}
int bottomEdge = cy;
gap = 0;
bool foundBottom = false;
int initialBridgeDown = Math.Max(maxGap * 2, 12);
for (int y = cy; y < h; y++)
{
int i = y * stride + x * 4;
int brightness = (px[i] + px[i + 1] + px[i + 2]) / 3;
if (brightness < darkThresh) { bottomEdge = y; gap = 0; foundBottom = true; }
else if (++gap > (foundBottom ? maxGap : initialBridgeDown)) break;
}
topExtents.Add(topEdge);
bottomExtents.Add(bottomEdge);
}
if (topExtents.Count < 10)
{
Log.Debug("edge-crop: too few dark columns ({Count})", topExtents.Count);
fullCapture.Dispose();
return null;
}
topExtents.Sort();
bottomExtents.Sort();
int topPctIdx = topExtents.Count / p.RowThreshDiv;
int botPctIdx = topExtents.Count * (p.ColThreshDiv - 1) / p.ColThreshDiv;
topPctIdx = Math.Clamp(topPctIdx, 0, topExtents.Count - 1);
botPctIdx = Math.Clamp(botPctIdx, 0, bottomExtents.Count - 1);
int bestRowStart = topExtents[topPctIdx];
int bestRowEnd = bottomExtents[botPctIdx];
if (bestRowEnd - bestRowStart + 1 < 50)
{
Log.Debug("edge-crop: vertical extent too small");
fullCapture.Dispose();
return null;
}
int minX = bestColStart;
int minY = bestRowStart;
int maxX = bestColEnd;
int maxY = bestRowEnd;
int rw = maxX - minX + 1;
int rh = maxY - minY + 1;
if (rw < 50 || rh < 50)
{
Log.Debug("edge-crop: region too small ({W}x{H})", rw, rh);
fullCapture.Dispose();
return null;
}
var cropRect = new Rectangle(minX, minY, rw, rh);
var cropped = fullCapture.Clone(cropRect, PixelFormat.Format32bppArgb);
var region = new Region(minX, minY, rw, rh);
return (cropped, fullCapture, region);
}
private static int FindDarkSeedInRow(byte[] px, int stride, int w, int rowOff, int cursorX, int darkThresh, int seedRadius)
{
int maxR = Math.Min(seedRadius, Math.Min(cursorX, w - 1 - cursorX));
for (int r = 0; r <= maxR; r++)
{
int x1 = cursorX - r;
int i1 = rowOff + x1 * 4;
int b1 = (px[i1] + px[i1 + 1] + px[i1 + 2]) / 3;
if (b1 < darkThresh) return x1;
int x2 = cursorX + r;
int i2 = rowOff + x2 * 4;
int b2 = (px[i2] + px[i2 + 1] + px[i2 + 2]) / 3;
if (b2 < darkThresh) return x2;
}
return -1;
}
private static int FindDarkSeedInColumn(byte[] px, int stride, int h, int x, int cursorY, int darkThresh, int seedRadius)
{
int maxR = Math.Min(seedRadius, Math.Min(cursorY, h - 1 - cursorY));
for (int r = 0; r <= maxR; r++)
{
int y1 = cursorY - r;
int i1 = y1 * stride + x * 4;
int b1 = (px[i1] + px[i1 + 1] + px[i1 + 2]) / 3;
if (b1 < darkThresh) return y1;
int y2 = cursorY + r;
int i2 = y2 * stride + x * 4;
int b2 = (px[i2] + px[i2 + 1] + px[i2 + 2]) / 3;
if (b2 < darkThresh) return y2;
}
return -1;
}
}

323
src/Poe2Trade.Screen/GridHandler.cs Normal file
View file

@ -0,0 +1,323 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using Serilog;
using Region = Poe2Trade.Core.Region;
public class GridHandler
{
private byte[]? _emptyTemplate70Gray;
private byte[]? _emptyTemplate70Argb;
private int _emptyTemplate70W, _emptyTemplate70H, _emptyTemplate70Stride;
private byte[]? _emptyTemplate35Gray;
private byte[]? _emptyTemplate35Argb;
private int _emptyTemplate35W, _emptyTemplate35H, _emptyTemplate35Stride;
public GridScanResult Scan(Region region, int cols, int rows,
int threshold = 0, int? targetRow = null, int? targetCol = null,
string? file = null, bool debug = false)
{
LoadTemplatesIfNeeded();
using var bitmap = ScreenCapture.CaptureOrLoad(file, region);
float cellW = (float)bitmap.Width / cols;
float cellH = (float)bitmap.Height / rows;
// Pick the right empty template based on cell size
int nominalCell = (int)Math.Round(cellW);
byte[]? templateGray;
byte[]? templateArgb;
int templateW, templateH, templateStride;
if (nominalCell <= 40 && _emptyTemplate35Gray != null)
{
templateGray = _emptyTemplate35Gray;
templateArgb = _emptyTemplate35Argb!;
templateW = _emptyTemplate35W;
templateH = _emptyTemplate35H;
templateStride = _emptyTemplate35Stride;
}
else if (_emptyTemplate70Gray != null)
{
templateGray = _emptyTemplate70Gray;
templateArgb = _emptyTemplate70Argb!;
templateW = _emptyTemplate70W;
templateH = _emptyTemplate70H;
templateStride = _emptyTemplate70Stride;
}
else
{
throw new InvalidOperationException("Empty cell templates not found in assets/");
}
var (captureGray, captureArgb, captureStride) = ImageUtils.BitmapToGrayAndArgb(bitmap);
int captureW = bitmap.Width;
int border = Math.Max(2, nominalCell / 10);
// Pre-compute template average for the inner region
long templateSum = 0;
int innerCount = 0;
for (int ty = border; ty < templateH - border; ty++)
for (int tx = border; tx < templateW - border; tx++)
{
templateSum += templateGray[ty * templateW + tx];
innerCount++;
}
double tmplMean = innerCount > 0 ? (double)templateSum / innerCount : 0;
double diffThreshold = threshold > 0 ? threshold : 5;
if (debug) Log.Debug("Grid: {Cols}x{Rows}, cellW={CellW:F1}, cellH={CellH:F1}, border={Border}, threshold={Threshold}, tmplMean={TmplMean:F1}",
cols, rows, cellW, cellH, border, diffThreshold, tmplMean);
var cells = new List<List<bool>>();
for (int row = 0; row < rows; row++)
{
var rowList = new List<bool>();
for (int col = 0; col < cols; col++)
{
int cx0 = (int)(col * cellW);
int cy0 = (int)(row * cellH);
int cw = (int)Math.Min(cellW, captureW - cx0);
int ch = (int)Math.Min(cellH, bitmap.Height - cy0);
int innerW = Math.Min(cw, templateW) - border;
int innerH = Math.Min(ch, templateH) - border;
long cellSum = 0;
int compared = 0;
for (int py = border; py < innerH; py++)
for (int px = border; px < innerW; px++)
{
cellSum += captureGray[(cy0 + py) * captureW + (cx0 + px)];
compared++;
}
double cellMean = compared > 0 ? (double)cellSum / compared : 0;
double offset = cellMean - tmplMean;
long diffSum = 0;
for (int py = border; py < innerH; py++)
for (int px = border; px < innerW; px++)
{
double cellVal = captureGray[(cy0 + py) * captureW + (cx0 + px)];
double tmplVal = templateGray[py * templateW + px];
diffSum += (long)Math.Abs(cellVal - tmplVal - offset);
}
double meanDiff = compared > 0 ? (double)diffSum / compared : 0;
bool occupied = meanDiff > diffThreshold;
rowList.Add(occupied);
}
cells.Add(rowList);
}
// Item detection: union-find on border pixel comparison
int[] parent = new int[rows * cols];
for (int i = 0; i < parent.Length; i++) parent[i] = i;
int Find(int x) { while (parent[x] != x) { parent[x] = parent[parent[x]]; x = parent[x]; } return x; }
void Union(int a, int b) { parent[Find(a)] = Find(b); }
int stripWidth = Math.Max(2, border / 2);
int stripInset = (int)(cellW * 0.15);
double borderDiffThresh = 15.0;
for (int row = 0; row < rows; row++)
{
for (int col = 0; col < cols; col++)
{
if (!cells[row][col]) continue;
int cx0 = (int)(col * cellW);
int cy0 = (int)(row * cellH);
// Check right neighbor
if (col + 1 < cols && cells[row][col + 1])
{
long diffSum = 0; int cnt = 0;
int xStart = (int)((col + 1) * cellW) - stripWidth;
int yFrom = cy0 + stripInset;
int yTo = (int)((row + 1) * cellH) - stripInset;
for (int sy = yFrom; sy < yTo; sy += 2)
{
int tmplY = sy - cy0;
for (int sx = xStart; sx < xStart + stripWidth * 2; sx++)
{
if (sx < 0 || sx >= captureW) continue;
int tmplX = sx - cx0;
if (tmplX < 0 || tmplX >= templateW) continue;
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
cnt++;
}
}
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
if (meanDiff > borderDiffThresh)
Union(row * cols + col, row * cols + col + 1);
}
// Check bottom neighbor
if (row + 1 < rows && cells[row + 1][col])
{
long diffSum = 0; int cnt = 0;
int yStart = (int)((row + 1) * cellH) - stripWidth;
int xFrom = cx0 + stripInset;
int xTo = (int)((col + 1) * cellW) - stripInset;
for (int sx = xFrom; sx < xTo; sx += 2)
{
int tmplX = sx - cx0;
for (int sy = yStart; sy < yStart + stripWidth * 2; sy++)
{
if (sy < 0 || sy >= bitmap.Height) continue;
int tmplY = sy - cy0;
if (tmplY < 0 || tmplY >= templateH) continue;
diffSum += Math.Abs(captureGray[sy * captureW + sx] - templateGray[tmplY * templateW + tmplX]);
cnt++;
}
}
double meanDiff = cnt > 0 ? (double)diffSum / cnt : 0;
if (meanDiff > borderDiffThresh)
Union(row * cols + col, (row + 1) * cols + col);
}
}
}
// Extract items from union-find groups
var groups = new Dictionary<int, List<(int row, int col)>>();
for (int row = 0; row < rows; row++)
for (int col = 0; col < cols; col++)
if (cells[row][col])
{
int root = Find(row * cols + col);
if (!groups.ContainsKey(root)) groups[root] = [];
groups[root].Add((row, col));
}
var items = new List<GridItem>();
foreach (var group in groups.Values)
{
int minR = group.Min(c => c.row);
int maxR = group.Max(c => c.row);
int minC = group.Min(c => c.col);
int maxC = group.Max(c => c.col);
items.Add(new GridItem { Row = minR, Col = minC, W = maxC - minC + 1, H = maxR - minR + 1 });
}
// Visual matching
List<GridMatch>? matches = null;
int tRow = targetRow ?? -1;
int tCol = targetCol ?? -1;
if (tRow >= 0 && tCol >= 0 && tRow < rows && tCol < cols && cells[tRow][tCol])
{
matches = FindMatchingCells(
captureGray, captureW, bitmap.Height,
cells, rows, cols, cellW, cellH, border,
tRow, tCol, debug);
}
// Convert cells to bool[][]
var cellsArr = cells.Select(r => r.ToArray()).ToArray();
return new GridScanResult { Cells = cellsArr, Items = items, Matches = matches };
}
private List<GridMatch> FindMatchingCells(
byte[] gray, int imgW, int imgH,
List<List<bool>> cells, int rows, int cols,
float cellW, float cellH, int border,
int targetRow, int targetCol, bool debug)
{
int innerW = (int)cellW - border * 2;
int innerH = (int)cellH - border * 2;
if (innerW <= 4 || innerH <= 4) return [];
int tCx0 = (int)(targetCol * cellW) + border;
int tCy0 = (int)(targetRow * cellH) + border;
int tInnerW = Math.Min(innerW, imgW - tCx0);
int tInnerH = Math.Min(innerH, imgH - tCy0);
if (tInnerW < innerW || tInnerH < innerH) return [];
int n = innerW * innerH;
double[] targetPixels = new double[n];
double tMean = 0;
for (int py = 0; py < innerH; py++)
for (int px = 0; px < innerW; px++)
{
double v = gray[(tCy0 + py) * imgW + (tCx0 + px)];
targetPixels[py * innerW + px] = v;
tMean += v;
}
tMean /= n;
double tStd = 0;
for (int i = 0; i < n; i++)
tStd += (targetPixels[i] - tMean) * (targetPixels[i] - tMean);
tStd = Math.Sqrt(tStd / n);
if (tStd < 3.0) return [];
double matchThreshold = 0.70;
var matches = new List<GridMatch>();
for (int row = 0; row < rows; row++)
{
for (int col = 0; col < cols; col++)
{
if (!cells[row][col]) continue;
if (row == targetRow && col == targetCol) continue;
int cx0 = (int)(col * cellW) + border;
int cy0 = (int)(row * cellH) + border;
int cInnerW = Math.Min(innerW, imgW - cx0);
int cInnerH = Math.Min(innerH, imgH - cy0);
if (cInnerW < innerW || cInnerH < innerH) continue;
double cMean = 0;
for (int py = 0; py < innerH; py++)
for (int px = 0; px < innerW; px++)
cMean += gray[(cy0 + py) * imgW + (cx0 + px)];
cMean /= n;
double cStd = 0, cross = 0;
for (int py = 0; py < innerH; py++)
for (int px = 0; px < innerW; px++)
{
double cv = gray[(cy0 + py) * imgW + (cx0 + px)] - cMean;
double tv = targetPixels[py * innerW + px] - tMean;
cStd += cv * cv;
cross += tv * cv;
}
cStd = Math.Sqrt(cStd / n);
double ncc = (tStd > 0 && cStd > 0) ? cross / (n * tStd * cStd) : 0;
if (ncc >= matchThreshold)
matches.Add(new GridMatch { Row = row, Col = col, Similarity = Math.Round(ncc, 3) });
}
}
return matches;
}
private void LoadTemplatesIfNeeded()
{
if (_emptyTemplate70Gray != null) return;
// Templates are in assets/ at project root (working directory)
var t70Path = Path.Combine("assets", "empty70.png");
var t35Path = Path.Combine("assets", "empty35.png");
if (File.Exists(t70Path))
{
using var bmp = new Bitmap(t70Path);
_emptyTemplate70W = bmp.Width;
_emptyTemplate70H = bmp.Height;
(_emptyTemplate70Gray, _emptyTemplate70Argb, _emptyTemplate70Stride) = ImageUtils.BitmapToGrayAndArgb(bmp);
}
if (File.Exists(t35Path))
{
using var bmp = new Bitmap(t35Path);
_emptyTemplate35W = bmp.Width;
_emptyTemplate35H = bmp.Height;
(_emptyTemplate35Gray, _emptyTemplate35Argb, _emptyTemplate35Stride) = ImageUtils.BitmapToGrayAndArgb(bmp);
}
}
}

136
src/Poe2Trade.Screen/GridReader.cs Normal file
View file

@ -0,0 +1,136 @@
using Poe2Trade.Core;
using Serilog;
namespace Poe2Trade.Screen;
public class GridLayout
{
public required Region Region { get; init; }
public required int Cols { get; init; }
public required int Rows { get; init; }
}
public record CellCoord(int Row, int Col, int X, int Y);
public class ScanResult
{
public required GridLayout Layout { get; init; }
public required List<CellCoord> Occupied { get; init; }
public required List<GridItem> Items { get; init; }
public List<GridMatch>? Matches { get; init; }
}
public static class GridLayouts
{
public static readonly Dictionary<string, GridLayout> All = new()
{
["inventory"] = new GridLayout
{
Region = new Region(1696, 788, 840, 350),
Cols = 12, Rows = 5
},
["stash12"] = new GridLayout
{
Region = new Region(23, 169, 840, 840),
Cols = 12, Rows = 12
},
["stash12_folder"] = new GridLayout
{
Region = new Region(23, 216, 840, 840),
Cols = 12, Rows = 12
},
["stash24"] = new GridLayout
{
Region = new Region(23, 169, 840, 840),
Cols = 24, Rows = 24
},
["stash24_folder"] = new GridLayout
{
Region = new Region(23, 216, 840, 840),
Cols = 24, Rows = 24
},
["seller"] = new GridLayout
{
Region = new Region(416, 299, 840, 840),
Cols = 12, Rows = 12
},
["shop"] = new GridLayout
{
Region = new Region(23, 216, 840, 840),
Cols = 12, Rows = 12
},
["vendor"] = new GridLayout
{
Region = new Region(416, 369, 840, 840),
Cols = 12, Rows = 12
},
};
public static readonly GridLayout Inventory = All["inventory"];
public static readonly GridLayout Seller = All["seller"];
}
public class GridReader
{
private readonly GridHandler _handler;
public GridReader(GridHandler handler)
{
_handler = handler;
}
public Task<ScanResult> Scan(string layoutName, int threshold = 0,
int? targetRow = null, int? targetCol = null)
{
if (!GridLayouts.All.TryGetValue(layoutName, out var layout))
throw new ArgumentException($"Unknown grid layout: {layoutName}");
var sw = System.Diagnostics.Stopwatch.StartNew();
var result = _handler.Scan(layout.Region, layout.Cols, layout.Rows,
threshold, targetRow, targetCol);
var occupied = new List<CellCoord>();
for (var row = 0; row < result.Cells.Length; row++)
for (var col = 0; col < result.Cells[row].Length; col++)
{
if (result.Cells[row][col])
{
var center = GetCellCenter(layout, row, col);
occupied.Add(new CellCoord(row, col, center.X, center.Y));
}
}
Log.Information("Grid scan {Layout}: {Occupied} occupied, {Items} items, {Ms}ms",
layoutName, occupied.Count, result.Items.Count, sw.ElapsedMilliseconds);
return Task.FromResult(new ScanResult
{
Layout = layout,
Occupied = occupied,
Items = result.Items,
Matches = result.Matches
});
}
public (int X, int Y) GetCellCenter(GridLayout layout, int row, int col)
{
var cellW = (double)layout.Region.Width / layout.Cols;
var cellH = (double)layout.Region.Height / layout.Rows;
return (
(int)Math.Round(layout.Region.X + col * cellW + cellW / 2),
(int)Math.Round(layout.Region.Y + row * cellH + cellH / 2)
);
}
public List<CellCoord> GetAllCells(GridLayout layout)
{
var cells = new List<CellCoord>();
for (var row = 0; row < layout.Rows; row++)
for (var col = 0; col < layout.Cols; col++)
{
var center = GetCellCenter(layout, row, col);
cells.Add(new CellCoord(row, col, center.X, center.Y));
}
return cells;
}
}

218
src/Poe2Trade.Screen/ImagePreprocessor.cs Normal file
View file

@ -0,0 +1,218 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using OpenCvSharp;
using OpenCvSharp.Extensions;
static class ImagePreprocessor
{
/// <summary>
/// Pre-process an image for OCR using morphological white top-hat filtering.
/// Isolates bright tooltip text, suppresses dim background text visible through overlay.
/// Pipeline: grayscale -> morphological top-hat -> Otsu binary -> upscale
/// </summary>
public static Bitmap PreprocessForOcr(Bitmap src, int kernelSize = 41, int upscale = 2)
{
using var mat = BitmapConverter.ToMat(src);
using var gray = new Mat();
Cv2.CvtColor(mat, gray, ColorConversionCodes.BGRA2GRAY);
// Morphological white top-hat: isolates bright text on dark background
using var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new OpenCvSharp.Size(kernelSize, kernelSize));
using var tophat = new Mat();
Cv2.MorphologyEx(gray, tophat, MorphTypes.TopHat, kernel);
// Otsu binarization: automatic threshold, black text on white
using var binary = new Mat();
Cv2.Threshold(tophat, binary, 0, 255, ThresholdTypes.BinaryInv | ThresholdTypes.Otsu);
// Upscale for better LSTM recognition
if (upscale > 1)
{
using var upscaled = new Mat();
Cv2.Resize(binary, upscaled, new OpenCvSharp.Size(binary.Width * upscale, binary.Height * upscale),
interpolation: InterpolationFlags.Cubic);
return BitmapConverter.ToBitmap(upscaled);
}
return BitmapConverter.ToBitmap(binary);
}
/// <summary>
/// Background-subtraction preprocessing: uses the reference frame to remove
/// background bleed-through from the semi-transparent tooltip overlay.
/// Pipeline: estimate dimming factor -> subtract expected background -> threshold -> upscale
/// Returns the upscaled binary Mat directly (caller must dispose).
/// </summary>
public static Mat PreprocessWithBackgroundSubMat(Bitmap tooltipCrop, Bitmap referenceCrop,
int dimPercentile = 25, int textThresh = 30, int upscale = 2, bool softThreshold = true)
{
using var curMat = BitmapConverter.ToMat(tooltipCrop);
using var refMat = BitmapConverter.ToMat(referenceCrop);
using var curGray = new Mat();
using var refGray = new Mat();
Cv2.CvtColor(curMat, curGray, ColorConversionCodes.BGRA2GRAY);
Cv2.CvtColor(refMat, refGray, ColorConversionCodes.BGRA2GRAY);
int rows = curGray.Rows, cols = curGray.Cols;
// Estimate the dimming factor of the tooltip overlay.
var ratios = new List<double>();
unsafe
{
byte* curPtr = (byte*)curGray.Data;
byte* refPtr = (byte*)refGray.Data;
int curStep = (int)curGray.Step();
int refStep = (int)refGray.Step();
for (int y = 0; y < rows; y++)
for (int x = 0; x < cols; x++)
{
byte r = refPtr[y * refStep + x];
byte c = curPtr[y * curStep + x];
if (r > 30)
ratios.Add((double)c / r);
}
}
if (ratios.Count == 0)
{
using var fallbackBmp = PreprocessForOcr(tooltipCrop, 41, upscale);
return BitmapConverter.ToMat(fallbackBmp);
}
ratios.Sort();
int idx = Math.Clamp(ratios.Count * dimPercentile / 100, 0, ratios.Count - 1);
double dimFactor = ratios[idx];
dimFactor = Math.Clamp(dimFactor, 0.05, 0.95);
// Subtract expected background: text_signal = current - reference * dimFactor
using var textSignal = new Mat(rows, cols, MatType.CV_8UC1);
unsafe
{
byte* curPtr = (byte*)curGray.Data;
byte* refPtr = (byte*)refGray.Data;
byte* outPtr = (byte*)textSignal.Data;
int curStep = (int)curGray.Step();
int refStep = (int)refGray.Step();
int outStep = (int)textSignal.Step();
for (int y = 0; y < rows; y++)
for (int x = 0; x < cols; x++)
{
double expected = refPtr[y * refStep + x] * dimFactor;
double signal = curPtr[y * curStep + x] - expected;
outPtr[y * outStep + x] = (byte)Math.Clamp(signal, 0, 255);
}
}
Mat result;
if (softThreshold)
{
result = new Mat(rows, cols, MatType.CV_8UC1);
unsafe
{
byte* srcPtr = (byte*)textSignal.Data;
byte* dstPtr = (byte*)result.Data;
int srcStep = (int)textSignal.Step();
int dstStep = (int)result.Step();
int maxClipped = 1;
for (int y = 0; y < rows; y++)
for (int x = 0; x < cols; x++)
{
int val = srcPtr[y * srcStep + x] - textThresh;
if (val > maxClipped) maxClipped = val;
}
for (int y = 0; y < rows; y++)
for (int x = 0; x < cols; x++)
{
int clipped = srcPtr[y * srcStep + x] - textThresh;
if (clipped <= 0)
{
dstPtr[y * dstStep + x] = 255;
}
else
{
int stretched = clipped * 255 / maxClipped;
dstPtr[y * dstStep + x] = (byte)(255 - stretched);
}
}
}
}
else
{
result = new Mat();
Cv2.Threshold(textSignal, result, textThresh, 255, ThresholdTypes.BinaryInv);
}
using var _result = result;
return UpscaleMat(result, upscale);
}
/// <summary>
/// Background-subtraction preprocessing returning a Bitmap (convenience wrapper).
/// </summary>
public static Bitmap PreprocessWithBackgroundSub(Bitmap tooltipCrop, Bitmap referenceCrop,
int dimPercentile = 25, int textThresh = 30, int upscale = 2, bool softThreshold = true)
{
using var mat = PreprocessWithBackgroundSubMat(tooltipCrop, referenceCrop, dimPercentile, textThresh, upscale, softThreshold);
return BitmapConverter.ToBitmap(mat);
}
/// <summary>
/// Detect text lines via horizontal projection on a binary image.
/// Binary should be inverted: text=black(0), background=white(255).
/// Returns list of (yStart, yEnd) row ranges for each detected text line.
/// </summary>
public static List<(int yStart, int yEnd)> DetectTextLines(
Mat binary, int minRowPixels = 2, int gapTolerance = 5)
{
int rows = binary.Rows, cols = binary.Cols;
var rowCounts = new int[rows];
unsafe
{
byte* ptr = (byte*)binary.Data;
int step = (int)binary.Step();
for (int y = 0; y < rows; y++)
for (int x = 0; x < cols; x++)
if (ptr[y * step + x] < 128)
rowCounts[y]++;
}
var lines = new List<(int yStart, int yEnd)>();
int lineStart = -1, lastActive = -1;
for (int y = 0; y < rows; y++)
{
if (rowCounts[y] >= minRowPixels)
{
if (lineStart < 0) lineStart = y;
lastActive = y;
}
else if (lineStart >= 0 && y - lastActive > gapTolerance)
{
lines.Add((lineStart, lastActive));
lineStart = -1;
}
}
if (lineStart >= 0)
lines.Add((lineStart, lastActive));
return lines;
}
/// <summary>Returns a new Mat (caller must dispose). Does NOT dispose src.</summary>
private static Mat UpscaleMat(Mat src, int factor)
{
if (factor > 1)
{
var upscaled = new Mat();
Cv2.Resize(src, upscaled, new OpenCvSharp.Size(src.Width * factor, src.Height * factor),
interpolation: InterpolationFlags.Cubic);
return upscaled;
}
return src.Clone();
}
}

39
src/Poe2Trade.Screen/ImageUtils.cs Normal file
View file

@ -0,0 +1,39 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using SdImageFormat = System.Drawing.Imaging.ImageFormat;
static class ImageUtils
{
public static (byte[] gray, byte[] argb, int stride) BitmapToGrayAndArgb(Bitmap bmp)
{
int w = bmp.Width, h = bmp.Height;
var data = bmp.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
byte[] argb = new byte[data.Stride * h];
Marshal.Copy(data.Scan0, argb, 0, argb.Length);
bmp.UnlockBits(data);
int stride = data.Stride;
byte[] gray = new byte[w * h];
for (int y = 0; y < h; y++)
for (int x = 0; x < w; x++)
{
int i = y * stride + x * 4;
gray[y * w + x] = (byte)((argb[i] + argb[i + 1] + argb[i + 2]) / 3);
}
return (gray, argb, stride);
}
public static SdImageFormat GetImageFormat(string path)
{
var ext = Path.GetExtension(path).ToLowerInvariant();
return ext switch
{
".jpg" or ".jpeg" => SdImageFormat.Jpeg,
".bmp" => SdImageFormat.Bmp,
_ => SdImageFormat.Png,
};
}
}

17
src/Poe2Trade.Screen/Poe2Trade.Screen.csproj Normal file
View file

@ -0,0 +1,17 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0-windows10.0.19041.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="OpenCvSharp4" Version="4.11.0.*" />
<PackageReference Include="OpenCvSharp4.Extensions" Version="4.11.0.*" />
<PackageReference Include="OpenCvSharp4.runtime.win" Version="4.11.0.*" />
<PackageReference Include="System.Drawing.Common" Version="8.0.12" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Poe2Trade.Core\Poe2Trade.Core.csproj" />
</ItemGroup>
</Project>

175
src/Poe2Trade.Screen/PythonOcrBridge.cs Normal file
View file

@ -0,0 +1,175 @@
namespace Poe2Trade.Screen;
using System.Diagnostics;
using System.Drawing;
using System.Text.Json;
using System.Text.Json.Serialization;
using Serilog;
using SdImageFormat = System.Drawing.Imaging.ImageFormat;
/// <summary>
/// Manages a persistent Python subprocess for EasyOCR.
/// Lazy-starts on first request; reuses the process for subsequent calls.
/// Same stdin/stdout JSON-per-line protocol.
/// </summary>
class PythonOcrBridge : IDisposable
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
};
private Process? _proc;
private readonly string _daemonScript;
private readonly string _pythonExe;
private readonly object _lock = new();
public PythonOcrBridge()
{
// Resolve paths relative to working directory
_daemonScript = Path.GetFullPath(Path.Combine("tools", "python-ocr", "daemon.py"));
var venvPython = Path.GetFullPath(Path.Combine("tools", "python-ocr", ".venv", "Scripts", "python.exe"));
_pythonExe = File.Exists(venvPython) ? venvPython : "python";
}
/// <summary>
/// Run OCR on a bitmap via the Python EasyOCR engine (base64 PNG over pipe).
/// </summary>
public OcrResponse OcrFromBitmap(Bitmap bitmap, OcrParams? ocrParams = null)
{
EnsureRunning();
using var ms = new MemoryStream();
bitmap.Save(ms, SdImageFormat.Png);
var imageBase64 = Convert.ToBase64String(ms.ToArray());
var pyReq = BuildPythonRequest(ocrParams);
pyReq["imageBase64"] = imageBase64;
return SendPythonRequest(pyReq);
}
private static Dictionary<string, object?> BuildPythonRequest(OcrParams? ocrParams)
{
var req = new Dictionary<string, object?> { ["cmd"] = "ocr", ["engine"] = "easyocr" };
if (ocrParams == null) return req;
if (ocrParams.MergeGap > 0) req["mergeGap"] = ocrParams.MergeGap;
if (ocrParams.LinkThreshold.HasValue) req["linkThreshold"] = ocrParams.LinkThreshold.Value;
if (ocrParams.TextThreshold.HasValue) req["textThreshold"] = ocrParams.TextThreshold.Value;
if (ocrParams.LowText.HasValue) req["lowText"] = ocrParams.LowText.Value;
if (ocrParams.WidthThs.HasValue) req["widthThs"] = ocrParams.WidthThs.Value;
if (ocrParams.Paragraph.HasValue) req["paragraph"] = ocrParams.Paragraph.Value;
return req;
}
private OcrResponse SendPythonRequest(object pyReq)
{
var json = JsonSerializer.Serialize(pyReq, JsonOptions);
string responseLine;
lock (_lock)
{
_proc!.StandardInput.WriteLine(json);
_proc.StandardInput.Flush();
responseLine = _proc.StandardOutput.ReadLine()
?? throw new Exception("Python daemon returned null");
}
var resp = JsonSerializer.Deserialize<PythonResponse>(responseLine, JsonOptions);
if (resp == null)
throw new Exception("Failed to parse Python OCR response");
if (!resp.Ok)
throw new Exception(resp.Error ?? "Python OCR failed");
return new OcrResponse
{
Text = resp.Text ?? "",
Lines = resp.Lines ?? [],
};
}
private void EnsureRunning()
{
if (_proc != null && !_proc.HasExited)
return;
_proc?.Dispose();
_proc = null;
if (!File.Exists(_daemonScript))
throw new Exception($"Python OCR daemon not found at {_daemonScript}");
Log.Information("Spawning Python OCR daemon: {Python} {Script}", _pythonExe, _daemonScript);
_proc = new Process
{
StartInfo = new ProcessStartInfo
{
FileName = _pythonExe,
Arguments = $"\"{_daemonScript}\"",
UseShellExecute = false,
RedirectStandardInput = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
}
};
_proc.ErrorDataReceived += (_, e) =>
{
if (!string.IsNullOrEmpty(e.Data))
Log.Debug("[python-ocr] {Line}", e.Data);
};
_proc.Start();
_proc.BeginErrorReadLine();
// Wait for ready signal (up to 30s for first model load)
var readyLine = _proc.StandardOutput.ReadLine();
if (readyLine == null)
throw new Exception("Python OCR daemon exited before ready signal");
var ready = JsonSerializer.Deserialize<PythonResponse>(readyLine, JsonOptions);
if (ready?.Ready != true)
throw new Exception($"Python OCR daemon did not send ready signal: {readyLine}");
Log.Information("Python OCR daemon ready");
}
public void Dispose()
{
if (_proc != null && !_proc.HasExited)
{
try
{
_proc.StandardInput.Close();
_proc.WaitForExit(3000);
if (!_proc.HasExited) _proc.Kill();
}
catch { /* ignore */ }
}
_proc?.Dispose();
_proc = null;
}
private class PythonResponse
{
[JsonPropertyName("ok")]
public bool Ok { get; set; }
[JsonPropertyName("ready")]
public bool? Ready { get; set; }
[JsonPropertyName("text")]
public string? Text { get; set; }
[JsonPropertyName("lines")]
public List<OcrLine>? Lines { get; set; }
[JsonPropertyName("error")]
public string? Error { get; set; }
}
}

66
src/Poe2Trade.Screen/ScreenCapture.cs Normal file
View file

@ -0,0 +1,66 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using Region = Poe2Trade.Core.Region;
static class ScreenCapture
{
[DllImport("user32.dll")]
private static extern bool SetProcessDPIAware();
[DllImport("user32.dll")]
private static extern int GetSystemMetrics(int nIndex);
public static void InitDpiAwareness() => SetProcessDPIAware();
/// <summary>
/// Capture from screen, or load from file if specified.
/// When file is set, loads the image and crops to region.
/// </summary>
public static Bitmap CaptureOrLoad(string? file, Region? region)
{
if (!string.IsNullOrEmpty(file))
{
var fullBmp = new Bitmap(file);
if (region != null)
{
int cx = Math.Max(0, region.X);
int cy = Math.Max(0, region.Y);
int cw = Math.Min(region.Width, fullBmp.Width - cx);
int ch = Math.Min(region.Height, fullBmp.Height - cy);
var cropped = fullBmp.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb);
fullBmp.Dispose();
return cropped;
}
return fullBmp;
}
return CaptureScreen(region);
}
public static Bitmap CaptureScreen(Region? region)
{
int x, y, w, h;
if (region != null)
{
x = region.X;
y = region.Y;
w = region.Width;
h = region.Height;
}
else
{
// Primary monitor only (0,0 origin, SM_CXSCREEN / SM_CYSCREEN)
x = 0;
y = 0;
w = GetSystemMetrics(0); // SM_CXSCREEN
h = GetSystemMetrics(1); // SM_CYSCREEN
}
var bitmap = new Bitmap(w, h, PixelFormat.Format32bppArgb);
using var g = Graphics.FromImage(bitmap);
g.CopyFromScreen(x, y, 0, 0, new System.Drawing.Size(w, h), CopyPixelOperation.SourceCopy);
return bitmap;
}
}

259
src/Poe2Trade.Screen/ScreenReader.cs Normal file
View file

@ -0,0 +1,259 @@
using Poe2Trade.Core;
using OpenCvSharp.Extensions;
using Serilog;
namespace Poe2Trade.Screen;
public class ScreenReader : IDisposable
{
private readonly DiffCropHandler _diffCrop = new();
private readonly GridHandler _gridHandler = new();
private readonly TemplateMatchHandler _templateMatch = new();
private readonly EdgeCropHandler _edgeCrop = new();
private readonly PythonOcrBridge _pythonBridge = new();
private bool _initialized;
public GridReader Grid { get; }
public ScreenReader()
{
Grid = new GridReader(_gridHandler);
}
public Task Warmup()
{
if (!_initialized)
{
ScreenCapture.InitDpiAwareness();
_initialized = true;
}
return Task.CompletedTask;
}
// -- Capture --
public Task<byte[]> CaptureScreen()
{
return Task.FromResult(_diffCrop.HandleCapture());
}
public Task<byte[]> CaptureRegion(Region region)
{
return Task.FromResult(_diffCrop.HandleCapture(region));
}
// -- OCR --
public Task<OcrResponse> Ocr(Region? region = null, string? preprocess = null)
{
using var bitmap = ScreenCapture.CaptureOrLoad(null, region);
if (preprocess == "tophat")
{
using var processed = ImagePreprocessor.PreprocessForOcr(bitmap);
return Task.FromResult(_pythonBridge.OcrFromBitmap(processed));
}
return Task.FromResult(_pythonBridge.OcrFromBitmap(bitmap));
}
public async Task<(int X, int Y)?> FindTextOnScreen(string searchText, bool fuzzy = false)
{
var result = await Ocr();
var pos = FindWordInOcrResult(result, searchText, fuzzy);
if (pos.HasValue)
Log.Information("Found text '{Text}' at ({X},{Y})", searchText, pos.Value.X, pos.Value.Y);
else
Log.Information("Text '{Text}' not found on screen", searchText);
return pos;
}
public async Task<string> ReadFullScreen()
{
var result = await Ocr();
return result.Text;
}
public async Task<(int X, int Y)?> FindTextInRegion(Region region, string searchText)
{
var result = await Ocr(region);
var pos = FindWordInOcrResult(result, searchText);
if (pos.HasValue)
return (region.X + pos.Value.X, region.Y + pos.Value.Y);
return null;
}
public async Task<string> ReadRegionText(Region region)
{
var result = await Ocr(region);
return result.Text;
}
public async Task<bool> CheckForText(Region region, string searchText)
{
var pos = await FindTextInRegion(region, searchText);
return pos.HasValue;
}
// -- Snapshot / Diff OCR --
public Task Snapshot()
{
_diffCrop.HandleSnapshot();
return Task.CompletedTask;
}
public Task<DiffOcrResponse> DiffOcr(string? savePath = null, Region? region = null)
{
var p = new DiffOcrParams();
var cropResult = _diffCrop.DiffCrop(p.Crop, region: region);
if (cropResult == null)
return Task.FromResult(new DiffOcrResponse { Text = "", Lines = [] });
var (cropped, refCropped, current, cropRegion) = cropResult.Value;
using var _current = current;
using var _cropped = cropped;
using var _refCropped = refCropped;
// Save raw crop if path is provided
if (!string.IsNullOrEmpty(savePath))
{
var dir = Path.GetDirectoryName(savePath);
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
Directory.CreateDirectory(dir);
cropped.Save(savePath, ImageUtils.GetImageFormat(savePath));
}
// Preprocess with background subtraction
var ocr = p.Ocr;
using var processedBmp = ocr.UseBackgroundSub
? ImagePreprocessor.PreprocessWithBackgroundSub(cropped, refCropped, ocr.DimPercentile, ocr.TextThresh, 1, ocr.SoftThreshold)
: ImagePreprocessor.PreprocessForOcr(cropped, ocr.KernelSize, 1);
var ocrResult = _pythonBridge.OcrFromBitmap(processedBmp, ocr);
// Offset coordinates to screen space
foreach (var line in ocrResult.Lines)
foreach (var word in line.Words)
{
word.X += cropRegion.X;
word.Y += cropRegion.Y;
}
return Task.FromResult(new DiffOcrResponse
{
Text = ocrResult.Text,
Lines = ocrResult.Lines,
Region = cropRegion,
});
}
// -- Template matching --
public Task<TemplateMatchResult?> TemplateMatch(string templatePath, Region? region = null)
{
var result = _templateMatch.Match(templatePath, region);
if (result != null)
Log.Information("Template match found: ({X},{Y}) confidence={Conf:F3}", result.X, result.Y, result.Confidence);
return Task.FromResult(result);
}
// -- Save --
public Task SaveScreenshot(string path)
{
_diffCrop.HandleScreenshot(path);
return Task.CompletedTask;
}
public Task SaveRegion(Region region, string path)
{
_diffCrop.HandleScreenshot(path, region);
return Task.CompletedTask;
}
public void Dispose() => _pythonBridge.Dispose();
// -- OCR text matching --
private static (int X, int Y)? FindWordInOcrResult(OcrResponse result, string needle, bool fuzzy = false)
{
var lower = needle.ToLowerInvariant();
const double fuzzyThreshold = 0.55;
if (lower.Contains(' '))
{
var needleNorm = Normalize(needle);
foreach (var line in result.Lines)
{
if (line.Words.Count == 0) continue;
if (line.Text.ToLowerInvariant().Contains(lower))
return LineBoundsCenter(line);
if (fuzzy)
{
var lineNorm = Normalize(line.Text);
var windowLen = needleNorm.Length;
for (var i = 0; i <= lineNorm.Length - windowLen + 2; i++)
{
var end = Math.Min(i + windowLen + 2, lineNorm.Length);
var window = lineNorm[i..end];
if (BigramSimilarity(needleNorm, window) >= fuzzyThreshold)
return LineBoundsCenter(line);
}
}
}
return null;
}
var needleN = Normalize(needle);
foreach (var line in result.Lines)
{
foreach (var word in line.Words)
{
if (word.Text.ToLowerInvariant().Contains(lower))
return (word.X + word.Width / 2, word.Y + word.Height / 2);
if (fuzzy && BigramSimilarity(needleN, Normalize(word.Text)) >= fuzzyThreshold)
return (word.X + word.Width / 2, word.Y + word.Height / 2);
}
}
return null;
}
private static (int X, int Y) LineBoundsCenter(OcrLine line)
{
var first = line.Words[0];
var last = line.Words[^1];
var x1 = first.X;
var y1 = first.Y;
var x2 = last.X + last.Width;
var y2 = line.Words.Max(w => w.Y + w.Height);
return ((x1 + x2) / 2, (y1 + y2) / 2);
}
private static string Normalize(string s) =>
new(s.ToLowerInvariant().Where(char.IsLetterOrDigit).ToArray());
private static double BigramSimilarity(string a, string b)
{
if (a.Length < 2 || b.Length < 2) return a == b ? 1 : 0;
var bigramsA = new Dictionary<string, int>();
for (var i = 0; i < a.Length - 1; i++)
{
var bg = a.Substring(i, 2);
bigramsA[bg] = bigramsA.GetValueOrDefault(bg) + 1;
}
var matches = 0;
for (var i = 0; i < b.Length - 1; i++)
{
var bg = b.Substring(i, 2);
if (bigramsA.TryGetValue(bg, out var count) && count > 0)
{
matches++;
bigramsA[bg] = count - 1;
}
}
return 2.0 * matches / (a.Length - 1 + b.Length - 1);
}
}

164
src/Poe2Trade.Screen/SignalProcessing.cs Normal file
View file

@ -0,0 +1,164 @@
namespace Poe2Trade.Screen;
static class SignalProcessing
{
/// <summary>
/// Find the dominant period in a signal using autocorrelation.
/// Returns (period, score) where score is the autocorrelation strength.
/// </summary>
public static (int period, double score) FindPeriodWithScore(double[] signal, int minPeriod, int maxPeriod)
{
int n = signal.Length;
if (n < minPeriod * 3) return (-1, 0);
double mean = signal.Average();
double variance = 0;
for (int i = 0; i < n; i++)
variance += (signal[i] - mean) * (signal[i] - mean);
if (variance < 1.0) return (-1, 0);
int maxLag = Math.Min(maxPeriod, n / 3);
double[] ac = new double[maxLag + 1];
for (int lag = minPeriod; lag <= maxLag; lag++)
{
double sum = 0;
for (int i = 0; i < n - lag; i++)
sum += (signal[i] - mean) * (signal[i + lag] - mean);
ac[lag] = sum / variance;
}
for (int lag = minPeriod + 1; lag < maxLag; lag++)
{
if (ac[lag] > 0.01 && ac[lag] >= ac[lag - 1] && ac[lag] >= ac[lag + 1])
return (lag, ac[lag]);
}
return (-1, 0);
}
/// <summary>
/// Find contiguous segments where values are ABOVE threshold.
/// </summary>
public static List<(int start, int end)> FindDarkDensitySegments(double[] profile, double threshold, int minLength)
{
var segments = new List<(int start, int end)>();
int n = profile.Length;
int curStart = -1;
int maxGap = 5;
int gapCount = 0;
for (int i = 0; i < n; i++)
{
if (profile[i] >= threshold)
{
if (curStart < 0) curStart = i;
gapCount = 0;
}
else
{
if (curStart >= 0)
{
gapCount++;
if (gapCount > maxGap)
{
int end = i - gapCount;
if (end - curStart >= minLength)
segments.Add((curStart, end));
curStart = -1;
gapCount = 0;
}
}
}
}
if (curStart >= 0)
{
int end = gapCount > 0 ? n - gapCount : n;
if (end - curStart >= minLength)
segments.Add((curStart, end));
}
return segments;
}
/// <summary>
/// Find the extent of the grid in a 1D profile using local autocorrelation.
/// </summary>
public static (int start, int end) FindGridExtent(double[] signal, int period)
{
int n = signal.Length;
int halfWin = period * 2;
if (n < halfWin * 2 + period) return (-1, -1);
double[] localAc = new double[n];
for (int center = halfWin; center < n - halfWin; center++)
{
int wStart = center - halfWin;
int wEnd = center + halfWin;
int count = wEnd - wStart;
double sum = 0;
for (int i = wStart; i < wEnd; i++)
sum += signal[i];
double mean = sum / count;
double varSum = 0;
for (int i = wStart; i < wEnd; i++)
varSum += (signal[i] - mean) * (signal[i] - mean);
if (varSum < 1.0) continue;
double acSum = 0;
for (int i = wStart; i < wEnd - period; i++)
acSum += (signal[i] - mean) * (signal[i + period] - mean);
localAc[center] = Math.Max(0, acSum / varSum);
}
double maxAc = 0;
for (int i = 0; i < n; i++)
if (localAc[i] > maxAc) maxAc = localAc[i];
if (maxAc < 0.02) return (-1, -1);
double threshold = maxAc * 0.25;
int bestStart = -1, bestEnd = -1, bestLen = 0;
int curStartPos = -1;
for (int i = 0; i < n; i++)
{
if (localAc[i] > threshold)
{
if (curStartPos < 0) curStartPos = i;
}
else
{
if (curStartPos >= 0)
{
int len = i - curStartPos;
if (len > bestLen)
{
bestLen = len;
bestStart = curStartPos;
bestEnd = i;
}
curStartPos = -1;
}
}
}
if (curStartPos >= 0)
{
int len = n - curStartPos;
if (len > bestLen)
{
bestStart = curStartPos;
bestEnd = n;
}
}
if (bestStart < 0) return (-1, -1);
bestStart = Math.Max(0, bestStart - period / 4);
bestEnd = Math.Min(n - 1, bestEnd + period / 4);
return (bestStart, bestEnd);
}
}

54
src/Poe2Trade.Screen/TemplateMatchHandler.cs Normal file
View file

@ -0,0 +1,54 @@
namespace Poe2Trade.Screen;
using System.Drawing;
using OpenCvSharp;
using OpenCvSharp.Extensions;
using Region = Poe2Trade.Core.Region;
class TemplateMatchHandler
{
public TemplateMatchResult? Match(string templatePath, Region? region = null,
string? file = null, double threshold = 0.7)
{
if (!System.IO.File.Exists(templatePath))
throw new FileNotFoundException($"Template file not found: {templatePath}");
using var screenshot = ScreenCapture.CaptureOrLoad(file, region);
using var screenMat = BitmapConverter.ToMat(screenshot);
using var template = Cv2.ImRead(templatePath, ImreadModes.Color);
if (template.Empty())
throw new InvalidOperationException($"Failed to load template image: {templatePath}");
// Convert screenshot from BGRA to BGR if needed
using var screenBgr = new Mat();
if (screenMat.Channels() == 4)
Cv2.CvtColor(screenMat, screenBgr, ColorConversionCodes.BGRA2BGR);
else
screenMat.CopyTo(screenBgr);
// Template must fit within screenshot
if (template.Rows > screenBgr.Rows || template.Cols > screenBgr.Cols)
return null;
using var result = new Mat();
Cv2.MatchTemplate(screenBgr, template, result, TemplateMatchModes.CCoeffNormed);
Cv2.MinMaxLoc(result, out _, out double maxVal, out _, out OpenCvSharp.Point maxLoc);
if (maxVal < threshold)
return null;
int offsetX = region?.X ?? 0;
int offsetY = region?.Y ?? 0;
return new TemplateMatchResult
{
X = offsetX + maxLoc.X + template.Cols / 2,
Y = offsetY + maxLoc.Y + template.Rows / 2,
Width = template.Cols,
Height = template.Rows,
Confidence = maxVal,
};
}
}