full capture
This commit is contained in:
parent
d71c1d97c5
commit
6ea373f2c3
10 changed files with 291 additions and 173 deletions
|
|
@ -5,68 +5,62 @@ using Size = OpenCvSharp.Size;
|
|||
|
||||
namespace Poe2Trade.Navigation;
|
||||
|
||||
public class MinimapCapture : IDisposable
|
||||
public class MinimapCapture : IFrameConsumer, IDisposable
|
||||
{
|
||||
private readonly MinimapConfig _config;
|
||||
private readonly IScreenCapture _backend;
|
||||
private readonly WallColorTracker _colorTracker;
|
||||
private readonly IScreenCapture _backend; // kept for debug capture paths
|
||||
private int _modeCheckCounter;
|
||||
private MinimapMode _detectedMode = MinimapMode.Overlay;
|
||||
private MinimapFrame? _lastFrame;
|
||||
|
||||
public MinimapMode DetectedMode => _detectedMode;
|
||||
public MinimapFrame? LastFrame => _lastFrame;
|
||||
public event Action<MinimapMode>? ModeChanged;
|
||||
|
||||
public MinimapCapture(MinimapConfig config)
|
||||
public MinimapCapture(MinimapConfig config, IScreenCapture backend)
|
||||
{
|
||||
_config = config;
|
||||
_colorTracker = new WallColorTracker(config.WallLoHSV, config.WallHiHSV);
|
||||
_backend = CreateBackend();
|
||||
_backend = backend;
|
||||
}
|
||||
|
||||
private static IScreenCapture CreateBackend()
|
||||
/// <summary>
|
||||
/// IFrameConsumer entry point — called by FramePipeline with the shared screen frame.
|
||||
/// </summary>
|
||||
public void Process(ScreenFrame screen)
|
||||
{
|
||||
// WGC primary → DXGI fallback → GDI last resort
|
||||
try
|
||||
// Auto-detect minimap mode every 10th frame via single pixel probe
|
||||
if (++_modeCheckCounter >= 10)
|
||||
{
|
||||
var wgc = new WgcCapture();
|
||||
Log.Information("Screen capture: WGC (Windows Graphics Capture)");
|
||||
return wgc;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning(ex, "WGC unavailable, trying DXGI Desktop Duplication");
|
||||
_modeCheckCounter = 0;
|
||||
var detected = DetectMinimapMode(screen);
|
||||
if (detected != _detectedMode)
|
||||
{
|
||||
_detectedMode = detected;
|
||||
Log.Information("Minimap mode switched to {Mode}", _detectedMode);
|
||||
ResetAdaptation();
|
||||
ModeChanged?.Invoke(_detectedMode);
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var dxgi = new DesktopDuplication();
|
||||
Log.Information("Screen capture: DXGI Desktop Duplication");
|
||||
return dxgi;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Warning(ex, "DXGI unavailable, falling back to GDI");
|
||||
}
|
||||
var region = _detectedMode == MinimapMode.Overlay
|
||||
? _config.OverlayRegion
|
||||
: _config.CornerRegion;
|
||||
|
||||
Log.Information("Screen capture: GDI (CopyFromScreen)");
|
||||
return new GdiCapture();
|
||||
using var bgr = screen.CropBgr(region);
|
||||
if (bgr.Empty())
|
||||
return;
|
||||
|
||||
var frame = ProcessBgr(bgr);
|
||||
if (frame == null) return;
|
||||
|
||||
var old = Interlocked.Exchange(ref _lastFrame, frame);
|
||||
old?.Dispose();
|
||||
}
|
||||
|
||||
public MinimapFrame? CaptureFrame()
|
||||
private MinimapFrame? ProcessBgr(Mat bgr)
|
||||
{
|
||||
// Auto-detect minimap mode every frame (just a 5x5 pixel check, negligible cost)
|
||||
var detected = DetectMinimapMode();
|
||||
if (detected != _detectedMode)
|
||||
{
|
||||
_detectedMode = detected;
|
||||
Log.Information("Minimap mode switched to {Mode}", _detectedMode);
|
||||
ResetAdaptation();
|
||||
ModeChanged?.Invoke(_detectedMode);
|
||||
}
|
||||
|
||||
using var bgr = CaptureAndNormalize(_detectedMode);
|
||||
if (bgr == null || bgr.Empty())
|
||||
return null;
|
||||
|
||||
using var hsv = new Mat();
|
||||
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
||||
|
||||
|
|
@ -84,12 +78,10 @@ public class MinimapCapture : IDisposable
|
|||
|
||||
if (_detectedMode == MinimapMode.Corner)
|
||||
{
|
||||
// Corner minimap is clean — skip fog detection
|
||||
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Overlay: fog detection needed (broad blue minus walls minus player)
|
||||
using var fogMask = BuildFogMask(hsv, wallMask, playerMask);
|
||||
classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask);
|
||||
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
|
||||
|
|
@ -101,7 +93,6 @@ public class MinimapCapture : IDisposable
|
|||
grayForCorr.SetTo(Scalar.Black, playerMask);
|
||||
|
||||
// Corner mode: rescale classified + wall mask to match overlay scale
|
||||
// Uses nearest-neighbor so discrete cell values (0-3) stay crisp
|
||||
if (_detectedMode == MinimapMode.Corner && Math.Abs(_config.CornerScale - 1.0) > 0.01)
|
||||
{
|
||||
var scaledSize = (int)Math.Round(frameSize * _config.CornerScale);
|
||||
|
|
@ -133,36 +124,37 @@ public class MinimapCapture : IDisposable
|
|||
);
|
||||
}
|
||||
|
||||
private Mat? CaptureAndNormalize(MinimapMode mode)
|
||||
{
|
||||
var region = mode == MinimapMode.Overlay
|
||||
? _config.OverlayRegion
|
||||
: _config.CornerRegion;
|
||||
|
||||
return _backend.CaptureRegion(region);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detect minimap mode by sampling a small patch at the corner minimap center.
|
||||
/// Detect minimap mode by sampling pixels at the corner minimap center.
|
||||
/// If the pixel is close to #DE581B (orange player dot), corner minimap is active.
|
||||
/// </summary>
|
||||
private MinimapMode DetectMinimapMode()
|
||||
private MinimapMode DetectMinimapMode(ScreenFrame screen)
|
||||
{
|
||||
// Capture a tiny 5x5 region at the corner center
|
||||
var cx = _config.CornerCenterX;
|
||||
var cy = _config.CornerCenterY;
|
||||
var probe = new Poe2Trade.Core.Region(cx - 2, cy - 2, 5, 5);
|
||||
using var patch = _backend.CaptureRegion(probe);
|
||||
if (patch == null || patch.Empty())
|
||||
return _detectedMode; // keep current on failure
|
||||
|
||||
// Average the BGR values of the patch
|
||||
var mean = Cv2.Mean(patch);
|
||||
var b = mean.Val0;
|
||||
var g = mean.Val1;
|
||||
var r = mean.Val2;
|
||||
// Bounds check
|
||||
if (cx < 2 || cy < 2 || cx + 2 >= screen.Width || cy + 2 >= screen.Height)
|
||||
return _detectedMode;
|
||||
|
||||
// #DE581B → R=222, G=88, B=27 — check if close (tolerance ~60 per channel)
|
||||
// Average a 5x5 patch worth of pixels
|
||||
double bSum = 0, gSum = 0, rSum = 0;
|
||||
var count = 0;
|
||||
for (var dy = -2; dy <= 2; dy++)
|
||||
for (var dx = -2; dx <= 2; dx++)
|
||||
{
|
||||
var px = screen.PixelAt(cx + dx, cy + dy);
|
||||
bSum += px.Item0;
|
||||
gSum += px.Item1;
|
||||
rSum += px.Item2;
|
||||
count++;
|
||||
}
|
||||
|
||||
var b = bSum / count;
|
||||
var g = gSum / count;
|
||||
var r = rSum / count;
|
||||
|
||||
// #DE581B → R=222, G=88, B=27
|
||||
const int tol = 60;
|
||||
if (Math.Abs(r - 222) < tol && Math.Abs(g - 88) < tol && Math.Abs(b - 27) < tol)
|
||||
return MinimapMode.Corner;
|
||||
|
|
@ -172,23 +164,19 @@ public class MinimapCapture : IDisposable
|
|||
|
||||
private Mat BuildWallMask(Mat hsv, Mat playerMask, bool sample = false)
|
||||
{
|
||||
// Use adapted range if available (narrows per-map), otherwise broad default
|
||||
var lo = _colorTracker.AdaptedLo ?? _config.WallLoHSV;
|
||||
var hi = _colorTracker.AdaptedHi ?? _config.WallHiHSV;
|
||||
|
||||
var wallMask = new Mat();
|
||||
Cv2.InRange(hsv, lo, hi, wallMask);
|
||||
|
||||
// Subtract player marker (orange overlaps blue range slightly on some maps)
|
||||
using var notPlayer = new Mat();
|
||||
Cv2.BitwiseNot(playerMask, notPlayer);
|
||||
Cv2.BitwiseAnd(wallMask, notPlayer, wallMask);
|
||||
|
||||
// Sample from pure color-selected pixels BEFORE morphological ops
|
||||
if (sample)
|
||||
_colorTracker.SampleFrame(hsv, wallMask);
|
||||
|
||||
// Dilate to connect thin wall line fragments
|
||||
using var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3));
|
||||
Cv2.Dilate(wallMask, wallMask, kernel);
|
||||
|
||||
|
|
@ -198,11 +186,9 @@ public class MinimapCapture : IDisposable
|
|||
|
||||
private Mat BuildFogMask(Mat hsv, Mat wallMask, Mat playerMask)
|
||||
{
|
||||
// Broad blue detection (captures walls + fog + any blue)
|
||||
using var allBlue = new Mat();
|
||||
Cv2.InRange(hsv, _config.FogLoHSV, _config.FogHiHSV, allBlue);
|
||||
|
||||
// Subtract player and walls → remaining blue is fog
|
||||
using var notPlayer = new Mat();
|
||||
Cv2.BitwiseNot(playerMask, notPlayer);
|
||||
Cv2.BitwiseAnd(allBlue, notPlayer, allBlue);
|
||||
|
|
@ -225,13 +211,12 @@ public class MinimapCapture : IDisposable
|
|||
using var centroids = new Mat();
|
||||
var numLabels = Cv2.ConnectedComponentsWithStats(mask, labels, stats, centroids);
|
||||
|
||||
if (numLabels <= 1) return; // only background
|
||||
if (numLabels <= 1) return;
|
||||
|
||||
// Clear mask, then re-add only components meeting min area
|
||||
mask.SetTo(Scalar.Black);
|
||||
for (var i = 1; i < numLabels; i++)
|
||||
{
|
||||
var area = stats.At<int>(i, 4); // CC_STAT_AREA
|
||||
var area = stats.At<int>(i, 4);
|
||||
if (area < minArea) continue;
|
||||
|
||||
using var comp = new Mat();
|
||||
|
|
@ -242,10 +227,15 @@ public class MinimapCapture : IDisposable
|
|||
|
||||
/// <summary>
|
||||
/// Capture a single frame and return the requested pipeline stage as PNG bytes.
|
||||
/// Uses CaptureRegion directly (debug path, not the pipeline).
|
||||
/// </summary>
|
||||
public byte[]? CaptureStage(MinimapDebugStage stage)
|
||||
{
|
||||
using var bgr = CaptureAndNormalize(_detectedMode);
|
||||
var region = _detectedMode == MinimapMode.Overlay
|
||||
? _config.OverlayRegion
|
||||
: _config.CornerRegion;
|
||||
|
||||
using var bgr = _backend.CaptureRegion(region);
|
||||
if (bgr == null || bgr.Empty()) return null;
|
||||
|
||||
if (stage == MinimapDebugStage.Raw) return EncodePng(bgr);
|
||||
|
|
@ -272,11 +262,10 @@ public class MinimapCapture : IDisposable
|
|||
using var fogMask = BuildFogMask(hsv, wallMask, playerMask);
|
||||
if (stage == MinimapDebugStage.Fog) return EncodePng(fogMask);
|
||||
|
||||
// Classified (walls + fog + player — explored is tracked by WorldMap)
|
||||
using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black);
|
||||
classified.SetTo(new Scalar(180, 140, 70), fogMask); // light blue for fog
|
||||
classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown for walls
|
||||
classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange for player
|
||||
classified.SetTo(new Scalar(180, 140, 70), fogMask);
|
||||
classified.SetTo(new Scalar(26, 45, 61), wallMask);
|
||||
classified.SetTo(new Scalar(0, 165, 255), playerMask);
|
||||
return EncodePng(classified);
|
||||
}
|
||||
|
||||
|
|
@ -292,7 +281,12 @@ public class MinimapCapture : IDisposable
|
|||
public void SaveDebugCapture(string dir = "debug-minimap")
|
||||
{
|
||||
Directory.CreateDirectory(dir);
|
||||
using var bgr = CaptureAndNormalize(_detectedMode);
|
||||
|
||||
var region = _detectedMode == MinimapMode.Overlay
|
||||
? _config.OverlayRegion
|
||||
: _config.CornerRegion;
|
||||
|
||||
using var bgr = _backend.CaptureRegion(region);
|
||||
if (bgr == null || bgr.Empty()) return;
|
||||
|
||||
using var hsv = new Mat();
|
||||
|
|
@ -303,17 +297,15 @@ public class MinimapCapture : IDisposable
|
|||
|
||||
using var wallMask = BuildWallMask(hsv, playerMask);
|
||||
|
||||
// Colorized classified (walls + player)
|
||||
using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black);
|
||||
classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown
|
||||
classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange
|
||||
classified.SetTo(new Scalar(26, 45, 61), wallMask);
|
||||
classified.SetTo(new Scalar(0, 165, 255), playerMask);
|
||||
|
||||
Cv2.ImWrite(Path.Combine(dir, "01-raw.png"), bgr);
|
||||
Cv2.ImWrite(Path.Combine(dir, "02-walls.png"), wallMask);
|
||||
Cv2.ImWrite(Path.Combine(dir, "03-player.png"), playerMask);
|
||||
Cv2.ImWrite(Path.Combine(dir, "04-classified.png"), classified);
|
||||
|
||||
// HSV channels
|
||||
var channels = Cv2.Split(hsv);
|
||||
Cv2.ImWrite(Path.Combine(dir, "05-hue.png"), channels[0]);
|
||||
Cv2.ImWrite(Path.Combine(dir, "06-sat.png"), channels[1]);
|
||||
|
|
@ -326,7 +318,7 @@ public class MinimapCapture : IDisposable
|
|||
private Point2d FindCentroid(Mat mask)
|
||||
{
|
||||
var moments = Cv2.Moments(mask, true);
|
||||
if (moments.M00 < 10) // not enough pixels
|
||||
if (moments.M00 < 10)
|
||||
return new Point2d(0, 0);
|
||||
|
||||
var cx = moments.M10 / moments.M00 - mask.Width / 2.0;
|
||||
|
|
@ -334,14 +326,11 @@ public class MinimapCapture : IDisposable
|
|||
return new Point2d(cx, cy);
|
||||
}
|
||||
|
||||
/// <summary>Commit pending wall color samples (call after successful template match).</summary>
|
||||
public void CommitWallColors() => _colorTracker.Commit();
|
||||
|
||||
/// <summary>Reset adaptive color tracking (call on area change).</summary>
|
||||
public void ResetAdaptation() => _colorTracker.Reset();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_backend.Dispose();
|
||||
_lastFrame?.Dispose();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue