From ec1f6274e38b18f4753d22f6e730935b411975f5 Mon Sep 17 00:00:00 2001 From: Boki Date: Fri, 13 Feb 2026 17:36:33 -0500 Subject: [PATCH] switching --- src/Poe2Trade.Navigation/MinimapCapture.cs | 112 +++++++++++-- .../NavigationExecutor.cs | 11 +- src/Poe2Trade.Navigation/NavigationTypes.cs | 42 ++++- src/Poe2Trade.Navigation/WorldMap.cs | 148 +++++++++++++----- .../ViewModels/MainWindowViewModel.cs | 2 +- 5 files changed, 246 insertions(+), 69 deletions(-) diff --git a/src/Poe2Trade.Navigation/MinimapCapture.cs b/src/Poe2Trade.Navigation/MinimapCapture.cs index d756c61..308f006 100644 --- a/src/Poe2Trade.Navigation/MinimapCapture.cs +++ b/src/Poe2Trade.Navigation/MinimapCapture.cs @@ -10,6 +10,10 @@ public class MinimapCapture : IDisposable private readonly MinimapConfig _config; private readonly IScreenCapture _backend; private readonly WallColorTracker _colorTracker; + private MinimapMode _detectedMode = MinimapMode.Overlay; + + public MinimapMode DetectedMode => _detectedMode; + public event Action? ModeChanged; public MinimapCapture(MinimapConfig config) { @@ -49,9 +53,17 @@ public class MinimapCapture : IDisposable public MinimapFrame? CaptureFrame() { - var region = _config.CaptureRegion; - using var bgr = _backend.CaptureRegion(region); + // Auto-detect minimap mode every frame (just a 5x5 pixel check, negligible cost) + var detected = DetectMinimapMode(); + if (detected != _detectedMode) + { + _detectedMode = detected; + Log.Information("Minimap mode switched to {Mode}", _detectedMode); + ResetAdaptation(); + ModeChanged?.Invoke(_detectedMode); + } + using var bgr = CaptureAndNormalize(_detectedMode); if (bgr == null || bgr.Empty()) return null; @@ -66,19 +78,52 @@ public class MinimapCapture : IDisposable // Wall mask: target #A2AEE5 blue-lavender structure lines (range adapts per-map) var wallMask = BuildWallMask(hsv, playerMask, sample: true); - // Fog of war: broad blue range minus walls minus player - using var fogMask = BuildFogMask(hsv, wallMask, playerMask); + // Build classified mat (use actual frame size — differs between overlay and corner) + var frameSize = bgr.Width; + var classified = new Mat(frameSize, frameSize, MatType.CV_8UC1, Scalar.Black); - // Build classified mat (fog first, walls override) - var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black); - classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask); - classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); + if (_detectedMode == MinimapMode.Corner) + { + // Corner minimap is clean — skip fog detection + classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); + } + else + { + // Overlay: fog detection needed (broad blue minus walls minus player) + using var fogMask = BuildFogMask(hsv, wallMask, playerMask); + classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask); + classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); + } // Gray for correlation tracking (player zeroed) var grayForCorr = new Mat(); Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY); grayForCorr.SetTo(Scalar.Black, playerMask); + // Corner mode: rescale classified + wall mask to match overlay scale + // Uses nearest-neighbor so discrete cell values (0-3) stay crisp + if (_detectedMode == MinimapMode.Corner && Math.Abs(_config.CornerScale - 1.0) > 0.01) + { + var scaledSize = (int)Math.Round(frameSize * _config.CornerScale); + var scaledClassified = new Mat(); + Cv2.Resize(classified, scaledClassified, new Size(scaledSize, scaledSize), + interpolation: InterpolationFlags.Nearest); + classified.Dispose(); + classified = scaledClassified; + + var scaledWalls = new Mat(); + Cv2.Resize(wallMask, scaledWalls, new Size(scaledSize, scaledSize), + interpolation: InterpolationFlags.Nearest); + wallMask.Dispose(); + wallMask = scaledWalls; + + var scaledGray = new Mat(); + Cv2.Resize(grayForCorr, scaledGray, new Size(scaledSize, scaledSize), + interpolation: InterpolationFlags.Linear); + grayForCorr.Dispose(); + grayForCorr = scaledGray; + } + return new MinimapFrame( GrayMat: grayForCorr, WallMask: wallMask, @@ -88,6 +133,43 @@ public class MinimapCapture : IDisposable ); } + private Mat? CaptureAndNormalize(MinimapMode mode) + { + var region = mode == MinimapMode.Overlay + ? _config.OverlayRegion + : _config.CornerRegion; + + return _backend.CaptureRegion(region); + } + + /// + /// Detect minimap mode by sampling a small patch at the corner minimap center. + /// If the pixel is close to #DE581B (orange player dot), corner minimap is active. + /// + private MinimapMode DetectMinimapMode() + { + // Capture a tiny 5x5 region at the corner center + var cx = _config.CornerCenterX; + var cy = _config.CornerCenterY; + var probe = new Poe2Trade.Core.Region(cx - 2, cy - 2, 5, 5); + using var patch = _backend.CaptureRegion(probe); + if (patch == null || patch.Empty()) + return _detectedMode; // keep current on failure + + // Average the BGR values of the patch + var mean = Cv2.Mean(patch); + var b = mean.Val0; + var g = mean.Val1; + var r = mean.Val2; + + // #DE581B → R=222, G=88, B=27 — check if close (tolerance ~60 per channel) + const int tol = 60; + if (Math.Abs(r - 222) < tol && Math.Abs(g - 88) < tol && Math.Abs(b - 27) < tol) + return MinimapMode.Corner; + + return MinimapMode.Overlay; + } + private Mat BuildWallMask(Mat hsv, Mat playerMask, bool sample = false) { // Use adapted range if available (narrows per-map), otherwise broad default @@ -163,8 +245,7 @@ public class MinimapCapture : IDisposable /// public byte[]? CaptureStage(MinimapDebugStage stage) { - var region = _config.CaptureRegion; - using var bgr = _backend.CaptureRegion(region); + using var bgr = CaptureAndNormalize(_detectedMode); if (bgr == null || bgr.Empty()) return null; if (stage == MinimapDebugStage.Raw) return EncodePng(bgr); @@ -192,7 +273,7 @@ public class MinimapCapture : IDisposable if (stage == MinimapDebugStage.Fog) return EncodePng(fogMask); // Classified (walls + fog + player — explored is tracked by WorldMap) - using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black); + using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black); classified.SetTo(new Scalar(180, 140, 70), fogMask); // light blue for fog classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown for walls classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange for player @@ -211,8 +292,7 @@ public class MinimapCapture : IDisposable public void SaveDebugCapture(string dir = "debug-minimap") { Directory.CreateDirectory(dir); - var region = _config.CaptureRegion; - using var bgr = _backend.CaptureRegion(region); + using var bgr = CaptureAndNormalize(_detectedMode); if (bgr == null || bgr.Empty()) return; using var hsv = new Mat(); @@ -224,7 +304,7 @@ public class MinimapCapture : IDisposable using var wallMask = BuildWallMask(hsv, playerMask); // Colorized classified (walls + player) - using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black); + using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black); classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange @@ -249,8 +329,8 @@ public class MinimapCapture : IDisposable if (moments.M00 < 10) // not enough pixels return new Point2d(0, 0); - var cx = moments.M10 / moments.M00 - _config.CaptureSize / 2.0; - var cy = moments.M01 / moments.M00 - _config.CaptureSize / 2.0; + var cx = moments.M10 / moments.M00 - mask.Width / 2.0; + var cy = moments.M01 / moments.M00 - mask.Height / 2.0; return new Point2d(cx, cy); } diff --git a/src/Poe2Trade.Navigation/NavigationExecutor.cs b/src/Poe2Trade.Navigation/NavigationExecutor.cs index e77c3e4..d7c3310 100644 --- a/src/Poe2Trade.Navigation/NavigationExecutor.cs +++ b/src/Poe2Trade.Navigation/NavigationExecutor.cs @@ -26,6 +26,12 @@ public class NavigationExecutor : IDisposable _config = config ?? new MinimapConfig(); _capture = new MinimapCapture(_config); _worldMap = new WorldMap(_config); + _capture.ModeChanged += _ => + { + _worldMap.Rebootstrap(); + _stuckCounter = 0; + _lastPosition = null; + }; } private void SetState(NavigationState s) @@ -81,7 +87,8 @@ public class NavigationExecutor : IDisposable } SetState(NavigationState.Processing); - var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask); + var mode = _capture.DetectedMode; + var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, mode); if (_worldMap.LastMatchSucceeded) _capture.CommitWallColors(); @@ -245,7 +252,7 @@ public class NavigationExecutor : IDisposable if (frame == null) return null; var stitchStart = sw.Elapsed.TotalMilliseconds; - var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask); + var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, _capture.DetectedMode); if (_worldMap.LastMatchSucceeded) _capture.CommitWallColors(); var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart; diff --git a/src/Poe2Trade.Navigation/NavigationTypes.cs b/src/Poe2Trade.Navigation/NavigationTypes.cs index 9cddf5d..d835c36 100644 --- a/src/Poe2Trade.Navigation/NavigationTypes.cs +++ b/src/Poe2Trade.Navigation/NavigationTypes.cs @@ -3,6 +3,12 @@ using OpenCvSharp; namespace Poe2Trade.Navigation; +public enum MinimapMode +{ + Overlay, // full-screen overlay (default, large) + Corner // tab minimap in top-right corner (small) +} + public enum NavigationState { Idle, @@ -56,17 +62,37 @@ public record MinimapFrame( public class MinimapConfig { - // Minimap center on screen (2560x1440) - public int MinimapCenterX { get; set; } = 1280; - public int MinimapCenterY { get; set; } = 700; + // Overlay minimap center on screen (2560x1440) — full-screen overlay + public int OverlayCenterX { get; set; } = 1280; + public int OverlayCenterY { get; set; } = 700; - // Capture region centered at minimap center - public Region CaptureRegion => new( - MinimapCenterX - CaptureSize / 2, - MinimapCenterY - CaptureSize / 2, + // Corner minimap (tab minimap, top-right) + public int CornerCenterX { get; set; } = 2370; + public int CornerCenterY { get; set; } = 189; + public int CornerRadius { get; set; } = 180; // extends 180px in each direction + + // Active capture size (pipeline operates at this resolution) + public int CaptureSize { get; set; } = 400; + + // Capture region for overlay mode + public Region OverlayRegion => new( + OverlayCenterX - CaptureSize / 2, + OverlayCenterY - CaptureSize / 2, CaptureSize, CaptureSize); - public int CaptureSize { get; set; } = 400; + // Capture region for corner mode + public Region CornerRegion => new( + CornerCenterX - CornerRadius, + CornerCenterY - CornerRadius, + CornerRadius * 2, CornerRadius * 2); + + // Scale factor: corner pixels → overlay pixels (tune until maps align across modes) + // Default assumes both show the same game area: overlay is 400px, corner is 360px + public double CornerScale { get; set; } = 1.0; + + // Legacy alias used by click-to-move + public int MinimapCenterX => OverlayCenterX; + public int MinimapCenterY => OverlayCenterY; // HSV range for player marker (orange X) public Scalar PlayerLoHSV { get; set; } = new(5, 80, 80); diff --git a/src/Poe2Trade.Navigation/WorldMap.cs b/src/Poe2Trade.Navigation/WorldMap.cs index 99faa28..3b7c454 100644 --- a/src/Poe2Trade.Navigation/WorldMap.cs +++ b/src/Poe2Trade.Navigation/WorldMap.cs @@ -13,6 +13,8 @@ public class WorldMap : IDisposable private int _frameCount; private int _consecutiveMatchFails; private Mat? _prevWallMask; // for frame deduplication + private bool _modeSwitchPending; // suppress blind bootstrap after mode switch + private bool _hasCanvasData; // true after first successful stitch public MapPosition Position => _position; public bool LastMatchSucceeded { get; private set; } @@ -29,21 +31,27 @@ public class WorldMap : IDisposable /// Match current wall mask against the accumulated map to find position, /// then stitch walls and paint explored area. /// - public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask) + public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask, MinimapMode mode = MinimapMode.Overlay) { var sw = Stopwatch.StartNew(); _frameCount++; - var needsBootstrap = _frameCount <= _config.WarmupFrames || _consecutiveMatchFails >= 30; + var isCorner = mode == MinimapMode.Corner; + var warmupFrames = isCorner ? 2 : _config.WarmupFrames; + // After mode switch, don't blindly stitch — wait for template match to confirm alignment + var needsBootstrap = !_modeSwitchPending + && (_frameCount <= warmupFrames || _consecutiveMatchFails >= 30); - // Block-based noise filter: zero out 50×50 blocks with >25% wall density - // Removes localized glow (waypoints, effects) while preserving real walls - var cleanFraction = FilterNoisyBlocks(wallMask, classifiedMat); - if (cleanFraction < 0.25 && !needsBootstrap) + // Block-based noise filter: only needed for overlay (game effects bleed through) + if (!isCorner) { - Log.Information("Noise filter: {Clean:P0} clean, skipping ({Ms:F1}ms)", - cleanFraction, sw.Elapsed.TotalMilliseconds); - return _position; + var cleanFraction = FilterNoisyBlocks(wallMask, classifiedMat); + if (cleanFraction < 0.25 && !needsBootstrap) + { + Log.Information("Noise filter: {Clean:P0} clean, skipping ({Ms:F1}ms)", + cleanFraction, sw.Elapsed.TotalMilliseconds); + return _position; + } } // Frame deduplication: skip if minimap hasn't scrolled yet @@ -69,7 +77,8 @@ public class WorldMap : IDisposable // Warmup / re-bootstrap: stitch at current position to seed the canvas if (needsBootstrap) { - StitchWithConfidence(classifiedMat, _position, boosted: true); + StitchWithConfidence(classifiedMat, _position, boosted: true, mode: mode); + _hasCanvasData = true; if (_consecutiveMatchFails >= 30) { Log.Information("Re-bootstrap: stitching at current position after {Fails} match failures ({Ms:F1}ms)", @@ -93,16 +102,43 @@ public class WorldMap : IDisposable { _consecutiveMatchFails++; LastMatchSucceeded = false; - Log.Information("MatchAndStitch: dedup={Dedup:F1}ms match={Match:F1}ms (FAILED x{Fails}) total={Total:F1}ms", - dedupMs, matchMs, _consecutiveMatchFails, sw.Elapsed.TotalMilliseconds); + if (_modeSwitchPending) + { + // After 60 failures, give up on cross-mode alignment and force bootstrap + if (_consecutiveMatchFails >= 60) + { + Log.Information("Mode switch: giving up on alignment after {Fails} tries, force bootstrap", + _consecutiveMatchFails); + _modeSwitchPending = false; + _consecutiveMatchFails = 0; + _frameCount = 0; // will trigger warmup bootstrap next frame + } + else + { + Log.Information("Mode switch: match failed x{Fails}, waiting for alignment ({Ms:F1}ms)", + _consecutiveMatchFails, sw.Elapsed.TotalMilliseconds); + } + } + else + { + Log.Information("MatchAndStitch: dedup={Dedup:F1}ms match={Match:F1}ms (FAILED x{Fails}) total={Total:F1}ms", + dedupMs, matchMs, _consecutiveMatchFails, sw.Elapsed.TotalMilliseconds); + } return _position; // don't stitch — wrong position would corrupt the canvas } + // Successful match — clear mode switch pending (alignment confirmed) + if (_modeSwitchPending) + { + Log.Information("Mode switch: alignment confirmed after {Fails} tries", _consecutiveMatchFails); + _modeSwitchPending = false; + } + _consecutiveMatchFails = 0; LastMatchSucceeded = true; _position = matched; var stitchStart = sw.Elapsed.TotalMilliseconds; - StitchWithConfidence(classifiedMat, _position, boosted: false); + StitchWithConfidence(classifiedMat, _position, boosted: false, mode: mode); var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart; Log.Information("MatchAndStitch: dedup={Dedup:F1}ms match={Match:F1}ms stitch={Stitch:F1}ms total={Total:F1}ms", @@ -173,9 +209,12 @@ public class WorldMap : IDisposable return new MapPosition(matchX, matchY); } - private void StitchWithConfidence(Mat classifiedMat, MapPosition position, bool boosted) + private void StitchWithConfidence(Mat classifiedMat, MapPosition position, bool boosted, + MinimapMode mode = MinimapMode.Overlay) { - var halfSize = _config.CaptureSize / 2; + var isCorner = mode == MinimapMode.Corner; + var frameSize = classifiedMat.Width; + var halfSize = frameSize / 2; var canvasX = (int)Math.Round(position.X) - halfSize; var canvasY = (int)Math.Round(position.Y) - halfSize; @@ -184,8 +223,8 @@ public class WorldMap : IDisposable var srcY = Math.Max(0, -canvasY); var dstX = Math.Max(0, canvasX); var dstY = Math.Max(0, canvasY); - var w = Math.Min(_config.CaptureSize - srcX, _config.CanvasSize - dstX); - var h = Math.Min(_config.CaptureSize - srcY, _config.CanvasSize - dstY); + var w = Math.Min(frameSize - srcX, _config.CanvasSize - dstX); + var h = Math.Min(frameSize - srcY, _config.CanvasSize - dstY); if (w <= 0 || h <= 0) return; @@ -196,15 +235,12 @@ public class WorldMap : IDisposable var dstRoi = new Mat(_canvas, dstRect); var confRoi = new Mat(_confidence, dstRect); - var confInc = (short)_config.ConfidenceInc; + // Corner minimap is clean — trust walls immediately, lower threshold + var confInc = isCorner ? (short)_config.ConfidenceMax : (short)_config.ConfidenceInc; var confDec = (short)_config.ConfidenceDec; - var confThreshold = (short)_config.ConfidenceThreshold; + var confThreshold = isCorner ? (short)2 : (short)_config.ConfidenceThreshold; var confMax = (short)_config.ConfidenceMax; - // Wall pixels: increase confidence. Non-wall pixels in visible area: decay confidence. - // Real walls accumulate high confidence (40) and survive brief non-confirmation during - // movement. Transient noise (waypoint glow, effects) only reaches moderate confidence - // and gets removed as it decays. for (var row = 0; row < h; row++) for (var col = 0; col < w; col++) { @@ -219,45 +255,59 @@ public class WorldMap : IDisposable } else if (conf > 0) { - // Visible area, not a wall → slow decay conf = Math.Max((short)(conf - confDec), (short)0); } else { - continue; // nothing to update + continue; } confRoi.Set(row, col, conf); var current = dstRoi.At(row, col); - // Explored→Wall needs double evidence (protects already-walked areas from noise) - var needed = current == (byte)MapCell.Explored + // Corner mode: no double-evidence needed (clean data) + var needed = !isCorner && current == (byte)MapCell.Explored ? (short)(confThreshold * 2) : confThreshold; if (conf >= needed) dstRoi.Set(row, col, (byte)MapCell.Wall); else if (current == (byte)MapCell.Wall && conf < confThreshold) - dstRoi.Set(row, col, (byte)MapCell.Explored); // lost confidence → demote + dstRoi.Set(row, col, (byte)MapCell.Explored); } - // Mark fog on canvas: only in a ring between ExploredRadius and ExploredRadius+5 - var fogInner2 = _config.ExploredRadius * _config.ExploredRadius; - var fogOuter = _config.ExploredRadius + 5; - var fogOuter2 = fogOuter * fogOuter; - - for (var row = 0; row < h; row++) - for (var col = 0; col < w; col++) + // Mark fog on canvas + if (isCorner) { - if (srcRoi.At(row, col) != (byte)MapCell.Fog) continue; + // Corner minimap: fog is clean, accept all fog pixels + for (var row = 0; row < h; row++) + for (var col = 0; col < w; col++) + { + if (srcRoi.At(row, col) != (byte)MapCell.Fog) continue; + if (dstRoi.At(row, col) == (byte)MapCell.Unknown) + dstRoi.Set(row, col, (byte)MapCell.Fog); + } + } + else + { + // Overlay: restrict fog to ring around player (filters spell effect noise) + var fogInner2 = _config.ExploredRadius * _config.ExploredRadius; + var fogOuter = _config.ExploredRadius + 5; + var fogOuter2 = fogOuter * fogOuter; - var fx = srcX + col - halfSize; - var fy = srcY + row - halfSize; - var dist2 = fx * fx + fy * fy; - if (dist2 < fogInner2 || dist2 > fogOuter2) continue; + for (var row = 0; row < h; row++) + for (var col = 0; col < w; col++) + { + if (srcRoi.At(row, col) != (byte)MapCell.Fog) continue; - if (dstRoi.At(row, col) == (byte)MapCell.Unknown) - dstRoi.Set(row, col, (byte)MapCell.Fog); + var fx = srcX + col - halfSize; + var fy = srcY + row - halfSize; + var dist2 = fx * fx + fy * fy; + if (dist2 < fogInner2 || dist2 > fogOuter2) continue; + + if (dstRoi.At(row, col) == (byte)MapCell.Unknown) + dstRoi.Set(row, col, (byte)MapCell.Fog); + } } // Mark explored area: circle around player, overwrite Unknown and Fog @@ -485,6 +535,20 @@ public class WorldMap : IDisposable _position = new MapPosition(_config.CanvasSize / 2.0, _config.CanvasSize / 2.0); _frameCount = 0; _consecutiveMatchFails = 0; + _hasCanvasData = false; + } + + /// + /// Re-bootstrap without clearing the canvas. Used when minimap mode switches + /// so new frames get stitched onto existing map data. + /// + public void Rebootstrap() + { + _prevWallMask?.Dispose(); + _prevWallMask = null; + _frameCount = 0; + _consecutiveMatchFails = 0; + _modeSwitchPending = _hasCanvasData; // only suppress if canvas has data worth protecting } public void Dispose() diff --git a/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs b/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs index 364abe5..0087502 100644 --- a/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs @@ -220,7 +220,7 @@ public partial class MainWindowViewModel : ObservableObject }); } - await Task.Delay(100, ct); + await Task.Delay(33, ct); // ~30 fps } catch (OperationCanceledException) { break; } catch (Exception ex)