boki/poe2-bot
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

boss getting close

Browse source
This commit is contained in:
Boki 2026-02-21 20:57:22 -05:00
parent f914443d86
commit aee3a7f22c
19 changed files with 422 additions and 119 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/Poe2Trade.Screen/BossDetector.cs
View file

@ -11,7 +11,8 @@ namespace Poe2Trade.Screen;
/// </summary>
public class BossDetector : IFrameConsumer, IDisposable
{
private const int MinConsecutiveFrames = 2;
private const int MinConsecutiveFrames = 1;
private const int MinConsecutiveMisses = 5; // don't clear _latest on a single miss frame
private const string ModelsDir = "tools/python-detect/models";
private OnnxYoloDetector? _detector;
@ -20,6 +21,7 @@ public class BossDetector : IFrameConsumer, IDisposable
private volatile BossSnapshot _latest = new([], 0, 0);
private BossSnapshot _previous = new([], 0, 0);
private int _consecutiveDetections;
private int _consecutiveMisses;
private int _inferenceCount;
// Async frame-slot: Process() drops frame here, background loop runs YOLO
@ -137,6 +139,9 @@ public class BossDetector : IFrameConsumer, IDisposable
old?.Dispose();
_frameReady.Reset();
_consecutiveDetections = 0;
_consecutiveMisses = 0;
_latest = new BossSnapshot([], 0, 0);
_previous = new BossSnapshot([], 0, 0);
}
private async Task InferenceLoop(CancellationToken ct)
@ -165,6 +170,7 @@ public class BossDetector : IFrameConsumer, IDisposable
if (detections.Count > 0)
{
_consecutiveDetections++;
_consecutiveMisses = 0;
var timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
var deltaMs = (float)(timestamp - _previous.Timestamp);
@ -197,8 +203,13 @@ public class BossDetector : IFrameConsumer, IDisposable
}
else
{
_consecutiveMisses++;
_consecutiveDetections = 0;
_latest = new BossSnapshot([], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), 0);
// Only clear after several consecutive misses to avoid
// flickering when YOLO drops a frame intermittently
if (_consecutiveMisses >= MinConsecutiveMisses)
_latest = new BossSnapshot([], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), 0);
}
}
finally

9
src/Poe2Trade.Screen/GridReader.cs
View file

@ -113,13 +113,18 @@ public class GridReader
});
}
private static readonly Random Rng = new();
public (int X, int Y) GetCellCenter(GridLayout layout, int row, int col)
{
var cellW = (double)layout.Region.Width / layout.Cols;
var cellH = (double)layout.Region.Height / layout.Rows;
// ±20% jitter within the cell so clicks aren't pixel-perfect
var jitterX = (int)(cellW * 0.2 * (Rng.NextDouble() * 2 - 1));
var jitterY = (int)(cellH * 0.2 * (Rng.NextDouble() * 2 - 1));
return (
(int)Math.Round(layout.Region.X + col * cellW + cellW / 2),
(int)Math.Round(layout.Region.Y + row * cellH + cellH / 2)
(int)Math.Round(layout.Region.X + col * cellW + cellW / 2) + jitterX,
(int)Math.Round(layout.Region.Y + row * cellH + cellH / 2) + jitterY
);
}

2
src/Poe2Trade.Screen/IScreenReader.cs
View file

@ -18,7 +18,7 @@ public interface IScreenReader : IDisposable
Task<DiffOcrResponse> DiffOcr(string? savePath = null, Region? region = null);
Task<TemplateMatchResult?> TemplateMatch(string templatePath, Region? region = null);
Task<List<TemplateMatchResult>> TemplateMatchAll(string templatePath, Region? region = null, double threshold = 0.7, bool silent = false);
Task<OcrResponse> NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
Task<OcrResponse> NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current, System.Drawing.Rectangle? scanRegion = null, string? savePath = null);
void SetLootBaseline(System.Drawing.Bitmap frame);
List<LootLabel> DetectLootLabels(System.Drawing.Bitmap reference, System.Drawing.Bitmap current);
System.Drawing.Bitmap CaptureRawBitmap();

99
src/Poe2Trade.Screen/ScreenReader.cs
View file

@ -212,16 +212,28 @@ public class ScreenReader : IScreenReader
// Nameplate search region — skip top HUD, bottom bar, and side margins
private const int NpTop = 120, NpBottom = 1080, NpMargin = 300;
public Task<OcrResponse> NameplateDiffOcr(Bitmap reference, Bitmap current)
public Task<OcrResponse> NameplateDiffOcr(Bitmap reference, Bitmap current, Rectangle? scanRegion = null, string? savePath = null)
{
int w = Math.Min(reference.Width, current.Width);
int h = Math.Min(reference.Height, current.Height);
// Clamp search region to image bounds
int scanY0 = Math.Min(NpTop, h);
int scanY1 = Math.Min(NpBottom, h);
int scanX0 = Math.Min(NpMargin, w);
int scanX1 = Math.Max(scanX0, w - NpMargin);
// Use provided scan region or fall back to default nameplate bounds
int scanY0, scanY1, scanX0, scanX1;
if (scanRegion.HasValue)
{
var r = scanRegion.Value;
scanY0 = Math.Clamp(r.Y, 0, h);
scanY1 = Math.Clamp(r.Y + r.Height, 0, h);
scanX0 = Math.Clamp(r.X, 0, w);
scanX1 = Math.Clamp(r.X + r.Width, 0, w);
}
else
{
scanY0 = Math.Min(NpTop, h);
scanY1 = Math.Min(NpBottom, h);
scanX0 = Math.Min(NpMargin, w);
scanX1 = Math.Max(scanX0, w - NpMargin);
}
var refData = reference.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb);
@ -238,6 +250,75 @@ public class ScreenReader : IScreenReader
const int brightThresh = 30;
int scanW = scanX1 - scanX0;
int scanH = scanY1 - scanY0;
// When a scan region is provided, just crop & diff-mask that region directly (no cluster stitching)
if (scanRegion.HasValue)
{
using var crop = new Bitmap(scanW, scanH, PixelFormat.Format32bppArgb);
var cropData = crop.LockBits(new Rectangle(0, 0, scanW, scanH), ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb);
byte[] cropPx = new byte[cropData.Stride * scanH];
int cropStride = cropData.Stride;
// Copy current image pixels, zeroing out any that didn't get brighter (diff mask)
Parallel.For(0, scanH, sy =>
{
int y = sy + scanY0;
int srcOff = y * stride;
int dstOff = sy * cropStride;
for (int sx = 0; sx < scanW; sx++)
{
int x = sx + scanX0;
int si = srcOff + x * 4;
int di = dstOff + sx * 4;
int brighter = (curPx[si] - refPx[si]) + (curPx[si + 1] - refPx[si + 1]) + (curPx[si + 2] - refPx[si + 2]);
if (brighter > brightThresh)
{
cropPx[di] = curPx[si];
cropPx[di + 1] = curPx[si + 1];
cropPx[di + 2] = curPx[si + 2];
cropPx[di + 3] = 255;
}
// else stays black (zeroed)
}
});
Marshal.Copy(cropPx, 0, cropData.Scan0, cropPx.Length);
crop.UnlockBits(cropData);
if (savePath != null)
{
try
{
var dir = Path.GetDirectoryName(savePath);
if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir);
crop.Save(savePath, System.Drawing.Imaging.ImageFormat.Png);
Log.Information("NameplateDiffOcr: saved crop to {Path}", savePath);
}
catch (Exception ex) { Log.Warning(ex, "NameplateDiffOcr: failed to save crop"); }
}
var ocrSw2 = System.Diagnostics.Stopwatch.StartNew();
OcrResponse ocrResult2;
try { ocrResult2 = _pythonBridge.OcrFromBitmap(crop); }
catch (TimeoutException)
{
Log.Warning("NameplateDiffOcr: crop OCR timed out");
return Task.FromResult(new OcrResponse { Text = "", Lines = [] });
}
Log.Information("NameplateDiffOcr: crop OCR in {Ms}ms", ocrSw2.ElapsedMilliseconds);
// Offset coordinates back to screen space
foreach (var line in ocrResult2.Lines)
foreach (var word in line.Words)
{
word.X += scanX0;
word.Y += scanY0;
}
return Task.FromResult(ocrResult2);
}
// Full-screen path: cluster detection + stitching
bool[] mask = new bool[scanW * scanH];
Parallel.For(0, scanH, sy =>
{
@ -321,9 +402,9 @@ public class ScreenReader : IScreenReader
foreach (var word in line.Words)
{
// Find which crop this word belongs to by Y position
var crop = crops.Last(c => word.Y >= c.stitchY);
word.X += crop.screenX;
word.Y = word.Y - crop.stitchY + crop.screenY;
var crop2 = crops.Last(c => word.Y >= c.stitchY);
word.X += crop2.screenX;
word.Y = word.Y - crop2.stitchY + crop2.screenY;
}
}