switching

This commit is contained in:
Boki 2026-02-13 17:36:33 -05:00
parent 490fb8bdba
commit ec1f6274e3
5 changed files with 246 additions and 69 deletions

View file

@ -10,6 +10,10 @@ public class MinimapCapture : IDisposable
private readonly MinimapConfig _config;
private readonly IScreenCapture _backend;
private readonly WallColorTracker _colorTracker;
private MinimapMode _detectedMode = MinimapMode.Overlay;
public MinimapMode DetectedMode => _detectedMode;
public event Action<MinimapMode>? ModeChanged;
public MinimapCapture(MinimapConfig config)
{
@ -49,9 +53,17 @@ public class MinimapCapture : IDisposable
public MinimapFrame? CaptureFrame()
{
var region = _config.CaptureRegion;
using var bgr = _backend.CaptureRegion(region);
// Auto-detect minimap mode every frame (just a 5x5 pixel check, negligible cost)
var detected = DetectMinimapMode();
if (detected != _detectedMode)
{
_detectedMode = detected;
Log.Information("Minimap mode switched to {Mode}", _detectedMode);
ResetAdaptation();
ModeChanged?.Invoke(_detectedMode);
}
using var bgr = CaptureAndNormalize(_detectedMode);
if (bgr == null || bgr.Empty())
return null;
@ -66,19 +78,52 @@ public class MinimapCapture : IDisposable
// Wall mask: target #A2AEE5 blue-lavender structure lines (range adapts per-map)
var wallMask = BuildWallMask(hsv, playerMask, sample: true);
// Fog of war: broad blue range minus walls minus player
using var fogMask = BuildFogMask(hsv, wallMask, playerMask);
// Build classified mat (use actual frame size — differs between overlay and corner)
var frameSize = bgr.Width;
var classified = new Mat(frameSize, frameSize, MatType.CV_8UC1, Scalar.Black);
// Build classified mat (fog first, walls override)
var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
if (_detectedMode == MinimapMode.Corner)
{
// Corner minimap is clean — skip fog detection
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
}
else
{
// Overlay: fog detection needed (broad blue minus walls minus player)
using var fogMask = BuildFogMask(hsv, wallMask, playerMask);
classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask);
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
}
// Gray for correlation tracking (player zeroed)
var grayForCorr = new Mat();
Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY);
grayForCorr.SetTo(Scalar.Black, playerMask);
// Corner mode: rescale classified + wall mask to match overlay scale
// Uses nearest-neighbor so discrete cell values (0-3) stay crisp
if (_detectedMode == MinimapMode.Corner && Math.Abs(_config.CornerScale - 1.0) > 0.01)
{
var scaledSize = (int)Math.Round(frameSize * _config.CornerScale);
var scaledClassified = new Mat();
Cv2.Resize(classified, scaledClassified, new Size(scaledSize, scaledSize),
interpolation: InterpolationFlags.Nearest);
classified.Dispose();
classified = scaledClassified;
var scaledWalls = new Mat();
Cv2.Resize(wallMask, scaledWalls, new Size(scaledSize, scaledSize),
interpolation: InterpolationFlags.Nearest);
wallMask.Dispose();
wallMask = scaledWalls;
var scaledGray = new Mat();
Cv2.Resize(grayForCorr, scaledGray, new Size(scaledSize, scaledSize),
interpolation: InterpolationFlags.Linear);
grayForCorr.Dispose();
grayForCorr = scaledGray;
}
return new MinimapFrame(
GrayMat: grayForCorr,
WallMask: wallMask,
@ -88,6 +133,43 @@ public class MinimapCapture : IDisposable
);
}
private Mat? CaptureAndNormalize(MinimapMode mode)
{
var region = mode == MinimapMode.Overlay
? _config.OverlayRegion
: _config.CornerRegion;
return _backend.CaptureRegion(region);
}
/// <summary>
/// Detect minimap mode by sampling a small patch at the corner minimap center.
/// If the pixel is close to #DE581B (orange player dot), corner minimap is active.
/// </summary>
private MinimapMode DetectMinimapMode()
{
// Capture a tiny 5x5 region at the corner center
var cx = _config.CornerCenterX;
var cy = _config.CornerCenterY;
var probe = new Poe2Trade.Core.Region(cx - 2, cy - 2, 5, 5);
using var patch = _backend.CaptureRegion(probe);
if (patch == null || patch.Empty())
return _detectedMode; // keep current on failure
// Average the BGR values of the patch
var mean = Cv2.Mean(patch);
var b = mean.Val0;
var g = mean.Val1;
var r = mean.Val2;
// #DE581B → R=222, G=88, B=27 — check if close (tolerance ~60 per channel)
const int tol = 60;
if (Math.Abs(r - 222) < tol && Math.Abs(g - 88) < tol && Math.Abs(b - 27) < tol)
return MinimapMode.Corner;
return MinimapMode.Overlay;
}
private Mat BuildWallMask(Mat hsv, Mat playerMask, bool sample = false)
{
// Use adapted range if available (narrows per-map), otherwise broad default
@ -163,8 +245,7 @@ public class MinimapCapture : IDisposable
/// </summary>
public byte[]? CaptureStage(MinimapDebugStage stage)
{
var region = _config.CaptureRegion;
using var bgr = _backend.CaptureRegion(region);
using var bgr = CaptureAndNormalize(_detectedMode);
if (bgr == null || bgr.Empty()) return null;
if (stage == MinimapDebugStage.Raw) return EncodePng(bgr);
@ -192,7 +273,7 @@ public class MinimapCapture : IDisposable
if (stage == MinimapDebugStage.Fog) return EncodePng(fogMask);
// Classified (walls + fog + player — explored is tracked by WorldMap)
using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black);
classified.SetTo(new Scalar(180, 140, 70), fogMask); // light blue for fog
classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown for walls
classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange for player
@ -211,8 +292,7 @@ public class MinimapCapture : IDisposable
public void SaveDebugCapture(string dir = "debug-minimap")
{
Directory.CreateDirectory(dir);
var region = _config.CaptureRegion;
using var bgr = _backend.CaptureRegion(region);
using var bgr = CaptureAndNormalize(_detectedMode);
if (bgr == null || bgr.Empty()) return;
using var hsv = new Mat();
@ -224,7 +304,7 @@ public class MinimapCapture : IDisposable
using var wallMask = BuildWallMask(hsv, playerMask);
// Colorized classified (walls + player)
using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black);
classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown
classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange
@ -249,8 +329,8 @@ public class MinimapCapture : IDisposable
if (moments.M00 < 10) // not enough pixels
return new Point2d(0, 0);
var cx = moments.M10 / moments.M00 - _config.CaptureSize / 2.0;
var cy = moments.M01 / moments.M00 - _config.CaptureSize / 2.0;
var cx = moments.M10 / moments.M00 - mask.Width / 2.0;
var cy = moments.M01 / moments.M00 - mask.Height / 2.0;
return new Point2d(cx, cy);
}

View file

@ -26,6 +26,12 @@ public class NavigationExecutor : IDisposable
_config = config ?? new MinimapConfig();
_capture = new MinimapCapture(_config);
_worldMap = new WorldMap(_config);
_capture.ModeChanged += _ =>
{
_worldMap.Rebootstrap();
_stuckCounter = 0;
_lastPosition = null;
};
}
private void SetState(NavigationState s)
@ -81,7 +87,8 @@ public class NavigationExecutor : IDisposable
}
SetState(NavigationState.Processing);
var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask);
var mode = _capture.DetectedMode;
var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, mode);
if (_worldMap.LastMatchSucceeded)
_capture.CommitWallColors();
@ -245,7 +252,7 @@ public class NavigationExecutor : IDisposable
if (frame == null) return null;
var stitchStart = sw.Elapsed.TotalMilliseconds;
var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask);
var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, _capture.DetectedMode);
if (_worldMap.LastMatchSucceeded)
_capture.CommitWallColors();
var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart;

View file

@ -3,6 +3,12 @@ using OpenCvSharp;
namespace Poe2Trade.Navigation;
public enum MinimapMode
{
Overlay, // full-screen overlay (default, large)
Corner // tab minimap in top-right corner (small)
}
public enum NavigationState
{
Idle,
@ -56,17 +62,37 @@ public record MinimapFrame(
public class MinimapConfig
{
// Minimap center on screen (2560x1440)
public int MinimapCenterX { get; set; } = 1280;
public int MinimapCenterY { get; set; } = 700;
// Overlay minimap center on screen (2560x1440) — full-screen overlay
public int OverlayCenterX { get; set; } = 1280;
public int OverlayCenterY { get; set; } = 700;
// Capture region centered at minimap center
public Region CaptureRegion => new(
MinimapCenterX - CaptureSize / 2,
MinimapCenterY - CaptureSize / 2,
// Corner minimap (tab minimap, top-right)
public int CornerCenterX { get; set; } = 2370;
public int CornerCenterY { get; set; } = 189;
public int CornerRadius { get; set; } = 180; // extends 180px in each direction
// Active capture size (pipeline operates at this resolution)
public int CaptureSize { get; set; } = 400;
// Capture region for overlay mode
public Region OverlayRegion => new(
OverlayCenterX - CaptureSize / 2,
OverlayCenterY - CaptureSize / 2,
CaptureSize, CaptureSize);
public int CaptureSize { get; set; } = 400;
// Capture region for corner mode
public Region CornerRegion => new(
CornerCenterX - CornerRadius,
CornerCenterY - CornerRadius,
CornerRadius * 2, CornerRadius * 2);
// Scale factor: corner pixels → overlay pixels (tune until maps align across modes)
// Default assumes both show the same game area: overlay is 400px, corner is 360px
public double CornerScale { get; set; } = 1.0;
// Legacy alias used by click-to-move
public int MinimapCenterX => OverlayCenterX;
public int MinimapCenterY => OverlayCenterY;
// HSV range for player marker (orange X)
public Scalar PlayerLoHSV { get; set; } = new(5, 80, 80);

View file

@ -13,6 +13,8 @@ public class WorldMap : IDisposable
private int _frameCount;
private int _consecutiveMatchFails;
private Mat? _prevWallMask; // for frame deduplication
private bool _modeSwitchPending; // suppress blind bootstrap after mode switch
private bool _hasCanvasData; // true after first successful stitch
public MapPosition Position => _position;
public bool LastMatchSucceeded { get; private set; }
@ -29,15 +31,20 @@ public class WorldMap : IDisposable
/// Match current wall mask against the accumulated map to find position,
/// then stitch walls and paint explored area.
/// </summary>
public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask)
public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask, MinimapMode mode = MinimapMode.Overlay)
{
var sw = Stopwatch.StartNew();
_frameCount++;
var needsBootstrap = _frameCount <= _config.WarmupFrames || _consecutiveMatchFails >= 30;
var isCorner = mode == MinimapMode.Corner;
var warmupFrames = isCorner ? 2 : _config.WarmupFrames;
// After mode switch, don't blindly stitch — wait for template match to confirm alignment
var needsBootstrap = !_modeSwitchPending
&& (_frameCount <= warmupFrames || _consecutiveMatchFails >= 30);
// Block-based noise filter: zero out 50×50 blocks with >25% wall density
// Removes localized glow (waypoints, effects) while preserving real walls
// Block-based noise filter: only needed for overlay (game effects bleed through)
if (!isCorner)
{
var cleanFraction = FilterNoisyBlocks(wallMask, classifiedMat);
if (cleanFraction < 0.25 && !needsBootstrap)
{
@ -45,6 +52,7 @@ public class WorldMap : IDisposable
cleanFraction, sw.Elapsed.TotalMilliseconds);
return _position;
}
}
// Frame deduplication: skip if minimap hasn't scrolled yet
if (_prevWallMask != null && _frameCount > 1)
@ -69,7 +77,8 @@ public class WorldMap : IDisposable
// Warmup / re-bootstrap: stitch at current position to seed the canvas
if (needsBootstrap)
{
StitchWithConfidence(classifiedMat, _position, boosted: true);
StitchWithConfidence(classifiedMat, _position, boosted: true, mode: mode);
_hasCanvasData = true;
if (_consecutiveMatchFails >= 30)
{
Log.Information("Re-bootstrap: stitching at current position after {Fails} match failures ({Ms:F1}ms)",
@ -93,16 +102,43 @@ public class WorldMap : IDisposable
{
_consecutiveMatchFails++;
LastMatchSucceeded = false;
if (_modeSwitchPending)
{
// After 60 failures, give up on cross-mode alignment and force bootstrap
if (_consecutiveMatchFails >= 60)
{
Log.Information("Mode switch: giving up on alignment after {Fails} tries, force bootstrap",
_consecutiveMatchFails);
_modeSwitchPending = false;
_consecutiveMatchFails = 0;
_frameCount = 0; // will trigger warmup bootstrap next frame
}
else
{
Log.Information("Mode switch: match failed x{Fails}, waiting for alignment ({Ms:F1}ms)",
_consecutiveMatchFails, sw.Elapsed.TotalMilliseconds);
}
}
else
{
Log.Information("MatchAndStitch: dedup={Dedup:F1}ms match={Match:F1}ms (FAILED x{Fails}) total={Total:F1}ms",
dedupMs, matchMs, _consecutiveMatchFails, sw.Elapsed.TotalMilliseconds);
}
return _position; // don't stitch — wrong position would corrupt the canvas
}
// Successful match — clear mode switch pending (alignment confirmed)
if (_modeSwitchPending)
{
Log.Information("Mode switch: alignment confirmed after {Fails} tries", _consecutiveMatchFails);
_modeSwitchPending = false;
}
_consecutiveMatchFails = 0;
LastMatchSucceeded = true;
_position = matched;
var stitchStart = sw.Elapsed.TotalMilliseconds;
StitchWithConfidence(classifiedMat, _position, boosted: false);
StitchWithConfidence(classifiedMat, _position, boosted: false, mode: mode);
var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart;
Log.Information("MatchAndStitch: dedup={Dedup:F1}ms match={Match:F1}ms stitch={Stitch:F1}ms total={Total:F1}ms",
@ -173,9 +209,12 @@ public class WorldMap : IDisposable
return new MapPosition(matchX, matchY);
}
private void StitchWithConfidence(Mat classifiedMat, MapPosition position, bool boosted)
private void StitchWithConfidence(Mat classifiedMat, MapPosition position, bool boosted,
MinimapMode mode = MinimapMode.Overlay)
{
var halfSize = _config.CaptureSize / 2;
var isCorner = mode == MinimapMode.Corner;
var frameSize = classifiedMat.Width;
var halfSize = frameSize / 2;
var canvasX = (int)Math.Round(position.X) - halfSize;
var canvasY = (int)Math.Round(position.Y) - halfSize;
@ -184,8 +223,8 @@ public class WorldMap : IDisposable
var srcY = Math.Max(0, -canvasY);
var dstX = Math.Max(0, canvasX);
var dstY = Math.Max(0, canvasY);
var w = Math.Min(_config.CaptureSize - srcX, _config.CanvasSize - dstX);
var h = Math.Min(_config.CaptureSize - srcY, _config.CanvasSize - dstY);
var w = Math.Min(frameSize - srcX, _config.CanvasSize - dstX);
var h = Math.Min(frameSize - srcY, _config.CanvasSize - dstY);
if (w <= 0 || h <= 0) return;
@ -196,15 +235,12 @@ public class WorldMap : IDisposable
var dstRoi = new Mat(_canvas, dstRect);
var confRoi = new Mat(_confidence, dstRect);
var confInc = (short)_config.ConfidenceInc;
// Corner minimap is clean — trust walls immediately, lower threshold
var confInc = isCorner ? (short)_config.ConfidenceMax : (short)_config.ConfidenceInc;
var confDec = (short)_config.ConfidenceDec;
var confThreshold = (short)_config.ConfidenceThreshold;
var confThreshold = isCorner ? (short)2 : (short)_config.ConfidenceThreshold;
var confMax = (short)_config.ConfidenceMax;
// Wall pixels: increase confidence. Non-wall pixels in visible area: decay confidence.
// Real walls accumulate high confidence (40) and survive brief non-confirmation during
// movement. Transient noise (waypoint glow, effects) only reaches moderate confidence
// and gets removed as it decays.
for (var row = 0; row < h; row++)
for (var col = 0; col < w; col++)
{
@ -219,29 +255,42 @@ public class WorldMap : IDisposable
}
else if (conf > 0)
{
// Visible area, not a wall → slow decay
conf = Math.Max((short)(conf - confDec), (short)0);
}
else
{
continue; // nothing to update
continue;
}
confRoi.Set(row, col, conf);
var current = dstRoi.At<byte>(row, col);
// Explored→Wall needs double evidence (protects already-walked areas from noise)
var needed = current == (byte)MapCell.Explored
// Corner mode: no double-evidence needed (clean data)
var needed = !isCorner && current == (byte)MapCell.Explored
? (short)(confThreshold * 2)
: confThreshold;
if (conf >= needed)
dstRoi.Set(row, col, (byte)MapCell.Wall);
else if (current == (byte)MapCell.Wall && conf < confThreshold)
dstRoi.Set(row, col, (byte)MapCell.Explored); // lost confidence → demote
dstRoi.Set(row, col, (byte)MapCell.Explored);
}
// Mark fog on canvas: only in a ring between ExploredRadius and ExploredRadius+5
// Mark fog on canvas
if (isCorner)
{
// Corner minimap: fog is clean, accept all fog pixels
for (var row = 0; row < h; row++)
for (var col = 0; col < w; col++)
{
if (srcRoi.At<byte>(row, col) != (byte)MapCell.Fog) continue;
if (dstRoi.At<byte>(row, col) == (byte)MapCell.Unknown)
dstRoi.Set(row, col, (byte)MapCell.Fog);
}
}
else
{
// Overlay: restrict fog to ring around player (filters spell effect noise)
var fogInner2 = _config.ExploredRadius * _config.ExploredRadius;
var fogOuter = _config.ExploredRadius + 5;
var fogOuter2 = fogOuter * fogOuter;
@ -259,6 +308,7 @@ public class WorldMap : IDisposable
if (dstRoi.At<byte>(row, col) == (byte)MapCell.Unknown)
dstRoi.Set(row, col, (byte)MapCell.Fog);
}
}
// Mark explored area: circle around player, overwrite Unknown and Fog
var pcx = (int)Math.Round(position.X);
@ -485,6 +535,20 @@ public class WorldMap : IDisposable
_position = new MapPosition(_config.CanvasSize / 2.0, _config.CanvasSize / 2.0);
_frameCount = 0;
_consecutiveMatchFails = 0;
_hasCanvasData = false;
}
/// <summary>
/// Re-bootstrap without clearing the canvas. Used when minimap mode switches
/// so new frames get stitched onto existing map data.
/// </summary>
public void Rebootstrap()
{
_prevWallMask?.Dispose();
_prevWallMask = null;
_frameCount = 0;
_consecutiveMatchFails = 0;
_modeSwitchPending = _hasCanvasData; // only suppress if canvas has data worth protecting
}
public void Dispose()

View file

@ -220,7 +220,7 @@ public partial class MainWindowViewModel : ObservableObject
});
}
await Task.Delay(100, ct);
await Task.Delay(33, ct); // ~30 fps
}
catch (OperationCanceledException) { break; }
catch (Exception ex)