From 40f013d07e02aebd44f781c6afae28ecd10a259c Mon Sep 17 00:00:00 2001 From: Boki Date: Fri, 13 Feb 2026 13:01:55 -0500 Subject: [PATCH] minimap --- src/Poe2Trade.Navigation/MinimapCapture.cs | 11 +- .../NavigationExecutor.cs | 32 ++++-- src/Poe2Trade.Navigation/NavigationTypes.cs | 9 +- src/Poe2Trade.Navigation/PositionTracker.cs | 103 ------------------ src/Poe2Trade.Navigation/WorldMap.cs | 103 ++++++++++++++++-- 5 files changed, 129 insertions(+), 129 deletions(-) delete mode 100644 src/Poe2Trade.Navigation/PositionTracker.cs diff --git a/src/Poe2Trade.Navigation/MinimapCapture.cs b/src/Poe2Trade.Navigation/MinimapCapture.cs index a7d0d62..511f900 100644 --- a/src/Poe2Trade.Navigation/MinimapCapture.cs +++ b/src/Poe2Trade.Navigation/MinimapCapture.cs @@ -68,22 +68,27 @@ public class MinimapCapture : IDisposable var playerOffset = FindCentroid(playerMask); // --- 3. Wall mask: bright OR saturated → structure lines --- - using var wallMask = BuildWallMask(satChan, valueChan, playerMask); + using var rawWallMask = BuildWallMask(satChan, valueChan, playerMask); // --- 4. Build classified mat (walls only — explored is tracked by WorldMap) --- var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black); - classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); + classified.SetTo(new Scalar((byte)MapCell.Wall), rawWallMask); // --- 5. Temporal smoothing: majority vote on walls --- var smoothed = TemporalSmooth(classified); // classified goes into ring buffer - // --- 7. Gray for phase correlation (player zeroed — it stays centered, walls shift with map) --- + // --- 6. Extract smoothed wall mask for tracking (filters transient noise) --- + var stableWallMask = new Mat(); + Cv2.Compare(smoothed, new Scalar((byte)MapCell.Wall), stableWallMask, CmpType.EQ); + + // --- 7. Gray for optical flow tracking (player zeroed) --- var grayForCorr = new Mat(); Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY); grayForCorr.SetTo(Scalar.Black, playerMask); return new MinimapFrame( GrayMat: grayForCorr, + WallMask: stableWallMask, ClassifiedMat: smoothed, PlayerOffset: playerOffset, Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() diff --git a/src/Poe2Trade.Navigation/NavigationExecutor.cs b/src/Poe2Trade.Navigation/NavigationExecutor.cs index 8fa3c73..3e1e20c 100644 --- a/src/Poe2Trade.Navigation/NavigationExecutor.cs +++ b/src/Poe2Trade.Navigation/NavigationExecutor.cs @@ -9,10 +9,11 @@ public class NavigationExecutor : IDisposable private readonly IGameController _game; private readonly MinimapConfig _config; private readonly MinimapCapture _capture; - private readonly PositionTracker _tracker; private readonly WorldMap _worldMap; private NavigationState _state = NavigationState.Idle; private bool _stopped; + private int _stuckCounter; + private MapPosition? _lastPosition; private static readonly Random Rng = new(); public event Action? StateChanged; @@ -23,7 +24,6 @@ public class NavigationExecutor : IDisposable _game = game; _config = config ?? new MinimapConfig(); _capture = new MinimapCapture(_config); - _tracker = new PositionTracker(_config); _worldMap = new WorldMap(_config); } @@ -43,9 +43,10 @@ public class NavigationExecutor : IDisposable public void Reset() { - _tracker.Reset(); _worldMap.Reset(); _stopped = false; + _stuckCounter = 0; + _lastPosition = null; SetState(NavigationState.Idle); } @@ -77,8 +78,19 @@ public class NavigationExecutor : IDisposable } SetState(NavigationState.Processing); - var pos = _tracker.UpdatePosition(frame.GrayMat); - _worldMap.StitchFrame(frame.ClassifiedMat, pos); + var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask); + + // Stuck detection: position hasn't moved enough over several frames + if (_lastPosition != null) + { + var dx = pos.X - _lastPosition.X; + var dy = pos.Y - _lastPosition.Y; + if (Math.Sqrt(dx * dx + dy * dy) < _config.StuckThreshold) + _stuckCounter++; + else + _stuckCounter = 0; + } + _lastPosition = pos; // 2. Movement decisions at slower rate var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); @@ -86,7 +98,7 @@ public class NavigationExecutor : IDisposable { lastMoveTime = now; - if (_tracker.IsStuck) + if (_stuckCounter >= _config.StuckFrameCount) { SetState(NavigationState.Stuck); Log.Information("Stuck detected, clicking random direction"); @@ -152,9 +164,9 @@ public class NavigationExecutor : IDisposable await ClickToMove(Math.Cos(angle), Math.Sin(angle)); } - public MapPosition Position => _tracker.Position; + public MapPosition Position => _worldMap.Position; public byte[] GetMapSnapshot() => _worldMap.GetMapSnapshot(); - public byte[] GetViewportSnapshot(int viewSize = 400) => _worldMap.GetViewportSnapshot(_tracker.Position, viewSize); + public byte[] GetViewportSnapshot(int viewSize = 400) => _worldMap.GetViewportSnapshot(_worldMap.Position, viewSize); /// /// Capture one frame, track position, stitch into world map. @@ -165,8 +177,7 @@ public class NavigationExecutor : IDisposable using var frame = _capture.CaptureFrame(); if (frame == null) return null; - var pos = _tracker.UpdatePosition(frame.GrayMat); - _worldMap.StitchFrame(frame.ClassifiedMat, pos); + var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask); if (stage == MinimapDebugStage.WorldMap) return _worldMap.GetViewportSnapshot(pos); @@ -179,7 +190,6 @@ public class NavigationExecutor : IDisposable public void Dispose() { _capture.Dispose(); - _tracker.Dispose(); _worldMap.Dispose(); } } diff --git a/src/Poe2Trade.Navigation/NavigationTypes.cs b/src/Poe2Trade.Navigation/NavigationTypes.cs index f6e6f31..87540e7 100644 --- a/src/Poe2Trade.Navigation/NavigationTypes.cs +++ b/src/Poe2Trade.Navigation/NavigationTypes.cs @@ -38,6 +38,7 @@ public enum MinimapDebugStage public record MinimapFrame( Mat GrayMat, + Mat WallMask, Mat ClassifiedMat, Point2d PlayerOffset, long Timestamp @@ -46,6 +47,7 @@ public record MinimapFrame( public void Dispose() { GrayMat.Dispose(); + WallMask.Dispose(); ClassifiedMat.Dispose(); } } @@ -92,8 +94,11 @@ public class MinimapConfig // World map canvas public int CanvasSize { get; set; } = 4000; - // Phase correlation confidence threshold - public double ConfidenceThreshold { get; set; } = 0.15; + // Template matching: search radius around current position estimate (pixels) + public int MatchSearchRadius { get; set; } = 50; + + // Template matching: minimum correlation confidence to accept a match + public double MatchConfidence { get; set; } = 0.3; // Stuck detection public double StuckThreshold { get; set; } = 2.0; diff --git a/src/Poe2Trade.Navigation/PositionTracker.cs b/src/Poe2Trade.Navigation/PositionTracker.cs deleted file mode 100644 index c4a2588..0000000 --- a/src/Poe2Trade.Navigation/PositionTracker.cs +++ /dev/null @@ -1,103 +0,0 @@ -using OpenCvSharp; -using Serilog; - -namespace Poe2Trade.Navigation; - -public class PositionTracker : IDisposable -{ - private readonly MinimapConfig _config; - private Mat? _prevGray; - private Mat? _hanningWindow; - private double _worldX; - private double _worldY; - private int _stuckCounter; - - public MapPosition Position => new(_worldX, _worldY); - public bool IsStuck => _stuckCounter >= _config.StuckFrameCount; - - public PositionTracker(MinimapConfig config) - { - _config = config; - _worldX = config.CanvasSize / 2.0; - _worldY = config.CanvasSize / 2.0; - } - - public MapPosition UpdatePosition(Mat currentGray) - { - if (_prevGray == null || _hanningWindow == null) - { - _prevGray = currentGray.Clone(); - _hanningWindow = new Mat(); - Cv2.CreateHanningWindow(_hanningWindow, currentGray.Size(), MatType.CV_64F); - return Position; - } - - // Convert to float64 - using var prev64 = new Mat(); - using var curr64 = new Mat(); - _prevGray.ConvertTo(prev64, MatType.CV_64F); - currentGray.ConvertTo(curr64, MatType.CV_64F); - - // High-pass filter: removes slow lighting changes, keeps edges/structure - using var prevBlur = new Mat(); - using var currBlur = new Mat(); - Cv2.GaussianBlur(prev64, prevBlur, new OpenCvSharp.Size(21, 21), 0); - Cv2.GaussianBlur(curr64, currBlur, new OpenCvSharp.Size(21, 21), 0); - using var prevHp = new Mat(); - using var currHp = new Mat(); - Cv2.Subtract(prev64, prevBlur, prevHp); - Cv2.Subtract(curr64, currBlur, currHp); - - var shift = Cv2.PhaseCorrelate(prevHp, currHp, _hanningWindow, out var confidence); - - if (confidence < _config.ConfidenceThreshold) - { - Log.Debug("Phase correlation low confidence: {Confidence:F3}", confidence); - _stuckCounter++; - _prevGray.Dispose(); - _prevGray = currentGray.Clone(); - return Position; - } - - // Negate: minimap scrolls opposite to player movement - var dx = -shift.X; - var dy = -shift.Y; - var displacement = Math.Sqrt(dx * dx + dy * dy); - - if (displacement < _config.StuckThreshold) - { - _stuckCounter++; - } - else - { - _stuckCounter = 0; - _worldX += dx; - _worldY += dy; - } - - Log.Debug("Position: ({X:F1}, {Y:F1}) dx={Dx:F1} dy={Dy:F1} conf={Conf:F3} stuck={Stuck}", - _worldX, _worldY, dx, dy, confidence, _stuckCounter); - - _prevGray.Dispose(); - _prevGray = currentGray.Clone(); - return Position; - } - - public void Reset() - { - _prevGray?.Dispose(); - _prevGray = null; - _hanningWindow?.Dispose(); - _hanningWindow = null; - _worldX = _config.CanvasSize / 2.0; - _worldY = _config.CanvasSize / 2.0; - _stuckCounter = 0; - Log.Information("Position tracker reset"); - } - - public void Dispose() - { - _prevGray?.Dispose(); - _hanningWindow?.Dispose(); - } -} diff --git a/src/Poe2Trade.Navigation/WorldMap.cs b/src/Poe2Trade.Navigation/WorldMap.cs index c779fe7..130643a 100644 --- a/src/Poe2Trade.Navigation/WorldMap.cs +++ b/src/Poe2Trade.Navigation/WorldMap.cs @@ -7,14 +7,100 @@ public class WorldMap : IDisposable { private readonly MinimapConfig _config; private readonly Mat _canvas; + private MapPosition _position; + private int _frameCount; + + public MapPosition Position => _position; public WorldMap(MinimapConfig config) { _config = config; _canvas = new Mat(config.CanvasSize, config.CanvasSize, MatType.CV_8UC1, Scalar.Black); + _position = new MapPosition(config.CanvasSize / 2.0, config.CanvasSize / 2.0); } - public void StitchFrame(Mat classifiedMat, MapPosition position) + /// + /// Match current wall mask against the accumulated map to find position, + /// then stitch walls and paint explored area. + /// + public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask) + { + _frameCount++; + + // First frame: just stitch at center + if (_frameCount <= 1) + { + Stitch(classifiedMat, _position); + return _position; + } + + // Match wallMask against canvas to find best position + var matched = MatchPosition(wallMask, _position); + if (matched != null) + _position = matched; + + Stitch(classifiedMat, _position); + return _position; + } + + private MapPosition? MatchPosition(Mat wallMask, MapPosition estimate) + { + var frameSize = wallMask.Width; + var searchPad = _config.MatchSearchRadius; + var searchSize = frameSize + 2 * searchPad; + + var cx = (int)Math.Round(estimate.X); + var cy = (int)Math.Round(estimate.Y); + + // Search region on canvas (centered on estimate) + var sx = cx - searchSize / 2; + var sy = cy - searchSize / 2; + + // Clamp to canvas bounds + var sx0 = Math.Max(0, sx); + var sy0 = Math.Max(0, sy); + var sx1 = Math.Min(_config.CanvasSize, sx + searchSize); + var sy1 = Math.Min(_config.CanvasSize, sy + searchSize); + var sw = sx1 - sx0; + var sh = sy1 - sy0; + + // Search region must be larger than template + if (sw <= frameSize || sh <= frameSize) + return null; + + // Extract search ROI and convert to binary wall mask + using var searchRoi = new Mat(_canvas, new Rect(sx0, sy0, sw, sh)); + using var canvasWalls = new Mat(); + Cv2.Compare(searchRoi, new Scalar((byte)MapCell.Wall), canvasWalls, CmpType.EQ); + + // Check if canvas has enough walls to match against + var canvasWallCount = Cv2.CountNonZero(canvasWalls); + if (canvasWallCount < 50) + return null; + + // Template match: find where frame's walls best align with canvas walls + using var result = new Mat(); + Cv2.MatchTemplate(canvasWalls, wallMask, result, TemplateMatchModes.CCoeffNormed); + Cv2.MinMaxLoc(result, out _, out var maxVal, out _, out var maxLoc); + + if (maxVal < _config.MatchConfidence) + { + Log.Debug("Map match low confidence: {Conf:F3}", maxVal); + return null; + } + + // Convert match position to world coordinates + // maxLoc is the top-left of the template in the search ROI + var matchX = sx0 + maxLoc.X + frameSize / 2.0; + var matchY = sy0 + maxLoc.Y + frameSize / 2.0; + + Log.Debug("Map match: ({X:F1}, {Y:F1}) conf={Conf:F3} walls={Walls}", + matchX, matchY, maxVal, canvasWallCount); + + return new MapPosition(matchX, matchY); + } + + private void Stitch(Mat classifiedMat, MapPosition position) { var halfSize = _config.CaptureSize / 2; var canvasX = (int)Math.Round(position.X) - halfSize; @@ -72,7 +158,6 @@ public class WorldMap : IDisposable var cx = (int)Math.Round(pos.X); var cy = (int)Math.Round(pos.Y); - // Scan in angular sectors to find direction with most Unknown cells at frontier var bestAngle = double.NaN; var bestScore = 0; const int sectorCount = 16; @@ -83,7 +168,6 @@ public class WorldMap : IDisposable var angle = 2 * Math.PI * sector / sectorCount; var score = 0; - // Sample along a cone in this direction, at the fog boundary and beyond for (var r = fogRadius - 20; r <= fogRadius + searchRadius; r += 5) { for (var spread = -15; spread <= 15; spread += 5) @@ -132,28 +216,25 @@ public class WorldMap : IDisposable var cy = (int)Math.Round(center.Y); var half = viewSize / 2; - // Clamp viewport to canvas var x0 = Math.Clamp(cx - half, 0, _config.CanvasSize - viewSize); var y0 = Math.Clamp(cy - half, 0, _config.CanvasSize - viewSize); var roi = new Mat(_canvas, new Rect(x0, y0, viewSize, viewSize)); - // Colorize: Unknown=#0d1117, Explored=#1f4068, Wall=#3d2d1a - using var colored = new Mat(viewSize, viewSize, MatType.CV_8UC3, new Scalar(23, 17, 13)); // BGR #0d1117 + using var colored = new Mat(viewSize, viewSize, MatType.CV_8UC3, new Scalar(23, 17, 13)); for (var r = 0; r < viewSize; r++) for (var c = 0; c < viewSize; c++) { var v = roi.At(r, c); if (v == (byte)MapCell.Explored) - colored.Set(r, c, new Vec3b(104, 64, 31)); // BGR #1f4068 + colored.Set(r, c, new Vec3b(104, 64, 31)); else if (v == (byte)MapCell.Wall) - colored.Set(r, c, new Vec3b(26, 45, 61)); // BGR #3d2d1a + colored.Set(r, c, new Vec3b(26, 45, 61)); } - // Draw player dot var px = cx - x0; var py = cy - y0; if (px >= 0 && px < viewSize && py >= 0 && py < viewSize) - Cv2.Circle(colored, new Point(px, py), 4, new Scalar(0, 140, 255), -1); // orange + Cv2.Circle(colored, new Point(px, py), 4, new Scalar(0, 140, 255), -1); Cv2.ImEncode(".png", colored, out var buf); return buf; @@ -162,6 +243,8 @@ public class WorldMap : IDisposable public void Reset() { _canvas.SetTo(Scalar.Black); + _position = new MapPosition(_config.CanvasSize / 2.0, _config.CanvasSize / 2.0); + _frameCount = 0; } public void Dispose()