diff --git a/assets/checkpoint-off.png b/assets/checkpoint-off.png new file mode 100644 index 0000000..261ebfa Binary files /dev/null and b/assets/checkpoint-off.png differ diff --git a/assets/checkpoint-on.png b/assets/checkpoint-on.png new file mode 100644 index 0000000..e0711cf Binary files /dev/null and b/assets/checkpoint-on.png differ diff --git a/assets/door.png b/assets/door.png new file mode 100644 index 0000000..f43748e Binary files /dev/null and b/assets/door.png differ diff --git a/src/Poe2Trade.Bot/BotOrchestrator.cs b/src/Poe2Trade.Bot/BotOrchestrator.cs index 4c4ec87..7796f2c 100644 --- a/src/Poe2Trade.Bot/BotOrchestrator.cs +++ b/src/Poe2Trade.Bot/BotOrchestrator.cs @@ -68,7 +68,7 @@ public class BotOrchestrator : IAsyncDisposable PipelineService = pipelineService; // Create consumers - var minimapCapture = new MinimapCapture(new MinimapConfig(), pipelineService.Backend); + var minimapCapture = new MinimapCapture(new MinimapConfig(), pipelineService.Backend, "assets"); GameState = new GameStateDetector(); HudReader = new HudReader(); EnemyDetector = new EnemyDetector(); diff --git a/src/Poe2Trade.Navigation/IconDetector.cs b/src/Poe2Trade.Navigation/IconDetector.cs new file mode 100644 index 0000000..f0f2b09 --- /dev/null +++ b/src/Poe2Trade.Navigation/IconDetector.cs @@ -0,0 +1,128 @@ +using OpenCvSharp; +using Serilog; + +namespace Poe2Trade.Navigation; + +/// +/// Detects minimap icons (doors, checkpoints) via template matching. +/// Loads RGBA templates once, converts to grayscale for matching. +/// +internal class IconDetector : IDisposable +{ + private readonly Mat _doorTemplate; + private readonly Mat _checkpointOffTemplate; + private readonly Mat _checkpointOnTemplate; + + private const double DoorThreshold = 0.65; + private const double CheckpointThreshold = 0.60; + + public IconDetector(string assetsDir) + { + _doorTemplate = LoadGray(Path.Combine(assetsDir, "door.png")); + _checkpointOffTemplate = LoadGray(Path.Combine(assetsDir, "checkpoint-off.png")); + _checkpointOnTemplate = LoadGray(Path.Combine(assetsDir, "checkpoint-on.png")); + + Log.Information("IconDetector loaded: door={DW}x{DH} cpOff={OW}x{OH} cpOn={NW}x{NH}", + _doorTemplate.Width, _doorTemplate.Height, + _checkpointOffTemplate.Width, _checkpointOffTemplate.Height, + _checkpointOnTemplate.Width, _checkpointOnTemplate.Height); + } + + private static Mat LoadGray(string path) + { + var bgra = Cv2.ImRead(path, ImreadModes.Unchanged); + if (bgra.Empty()) + throw new FileNotFoundException($"Icon template not found: {path}"); + + var gray = new Mat(); + if (bgra.Channels() == 4) + { + using var bgr = new Mat(); + Cv2.CvtColor(bgra, bgr, ColorConversionCodes.BGRA2BGR); + Cv2.CvtColor(bgr, gray, ColorConversionCodes.BGR2GRAY); + } + else if (bgra.Channels() == 3) + { + Cv2.CvtColor(bgra, gray, ColorConversionCodes.BGR2GRAY); + } + else + { + gray = bgra.Clone(); + } + + bgra.Dispose(); + return gray; + } + + /// + /// Detect all door icons in the frame. Returns center points in frame coords. + /// + public List DetectDoors(Mat grayFrame) + => Detect(grayFrame, _doorTemplate, DoorThreshold); + + /// + /// Detect all inactive checkpoint icons. Returns center points in frame coords. + /// + public List DetectCheckpointsOff(Mat grayFrame) + => Detect(grayFrame, _checkpointOffTemplate, CheckpointThreshold); + + /// + /// Detect all active checkpoint icons. Returns center points in frame coords. + /// + public List DetectCheckpointsOn(Mat grayFrame) + => Detect(grayFrame, _checkpointOnTemplate, CheckpointThreshold); + + /// + /// Greedy multi-match: find best match, record center, zero out neighborhood, repeat. + /// + private static List Detect(Mat grayFrame, Mat template, double threshold) + { + var results = new List(); + + if (grayFrame.Width < template.Width || grayFrame.Height < template.Height) + return results; + + using var result = new Mat(); + Cv2.MatchTemplate(grayFrame, template, result, TemplateMatchModes.CCoeffNormed); + + var tw = template.Width; + var th = template.Height; + + while (true) + { + Cv2.MinMaxLoc(result, out _, out var maxVal, out _, out var maxLoc); + if (maxVal < threshold) + break; + + // Record center point of the matched region + var centerX = maxLoc.X + tw / 2; + var centerY = maxLoc.Y + th / 2; + results.Add(new Point(centerX, centerY)); + + // Zero out neighborhood to prevent re-detection + var zeroX = Math.Max(0, maxLoc.X - tw / 2); + var zeroY = Math.Max(0, maxLoc.Y - th / 2); + var zeroW = Math.Min(tw * 2, result.Width - zeroX); + var zeroH = Math.Min(th * 2, result.Height - zeroY); + if (zeroW > 0 && zeroH > 0) + { + using var roi = new Mat(result, new Rect(zeroX, zeroY, zeroW, zeroH)); + roi.SetTo(Scalar.Black); + } + } + + return results; + } + + /// + /// Get the template size for doors (used to stamp wall regions). + /// + public Size DoorSize => new(_doorTemplate.Width, _doorTemplate.Height); + + public void Dispose() + { + _doorTemplate.Dispose(); + _checkpointOffTemplate.Dispose(); + _checkpointOnTemplate.Dispose(); + } +} diff --git a/src/Poe2Trade.Navigation/MinimapCapture.cs b/src/Poe2Trade.Navigation/MinimapCapture.cs index 764e3b7..32981a3 100644 --- a/src/Poe2Trade.Navigation/MinimapCapture.cs +++ b/src/Poe2Trade.Navigation/MinimapCapture.cs @@ -11,6 +11,7 @@ public class MinimapCapture : IFrameConsumer, IDisposable private readonly MinimapConfig _config; private readonly WallColorTracker _colorTracker; private readonly IScreenCapture _backend; // kept for debug capture paths + private readonly IconDetector? _iconDetector; private int _modeCheckCounter = 9; // trigger mode detection on first frame private MinimapMode _detectedMode = MinimapMode.Overlay; private int _pendingModeCount; // consecutive detections of a different mode @@ -21,11 +22,18 @@ public class MinimapCapture : IFrameConsumer, IDisposable public MinimapFrame? LastFrame => _lastFrame; public event Action? ModeChanged; - public MinimapCapture(MinimapConfig config, IScreenCapture backend) + public MinimapCapture(MinimapConfig config, IScreenCapture backend, string? assetsDir = null) { _config = config; _colorTracker = new WallColorTracker(config.WallLoHSV, config.WallHiHSV); _backend = backend; + + // Load icon templates if assets directory provided and contains templates + if (assetsDir != null && File.Exists(Path.Combine(assetsDir, "door.png"))) + { + try { _iconDetector = new IconDetector(assetsDir); } + catch (Exception ex) { Log.Warning(ex, "Failed to load icon templates, icon detection disabled"); } + } } /// @@ -115,17 +123,50 @@ public class MinimapCapture : IFrameConsumer, IDisposable var frameSize = bgr.Width; var classified = new Mat(frameSize, frameSize, MatType.CV_8UC1, Scalar.Black); - if (_detectedMode == MinimapMode.Corner) - { - classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); - } - else { using var fogMask = BuildFogMask(hsv, wallMask, playerMask); classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask); classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); } + // Icon detection (overlay mode only — corner has different scale) + List? checkpointsOff = null; + List? checkpointsOn = null; + if (_detectedMode == MinimapMode.Overlay && _iconDetector != null) + { + using var grayForIcons = new Mat(); + Cv2.CvtColor(bgr, grayForIcons, ColorConversionCodes.BGR2GRAY); + + // Doors: stamp as Wall in classified + wallMask (fills gap so BFS blocks doors) + var doors = _iconDetector.DetectDoors(grayForIcons); + if (doors.Count > 0) + { + var doorSize = _iconDetector.DoorSize; + const int pad = 2; + foreach (var d in doors) + { + var rx = Math.Max(0, d.X - doorSize.Width / 2 - pad); + var ry = Math.Max(0, d.Y - doorSize.Height / 2 - pad); + var rw = Math.Min(doorSize.Width + 2 * pad, frameSize - rx); + var rh = Math.Min(doorSize.Height + 2 * pad, frameSize - ry); + if (rw <= 0 || rh <= 0) continue; + + var rect = new Rect(rx, ry, rw, rh); + using var classRoi = new Mat(classified, rect); + classRoi.SetTo(new Scalar((byte)MapCell.Wall)); + using var wallRoi = new Mat(wallMask, rect); + wallRoi.SetTo(new Scalar(255)); + } + Log.Debug("Icons: {Count} doors stamped as wall", doors.Count); + } + + // Checkpoints: don't stamp — just pass positions into MinimapFrame + checkpointsOff = _iconDetector.DetectCheckpointsOff(grayForIcons); + checkpointsOn = _iconDetector.DetectCheckpointsOn(grayForIcons); + if (checkpointsOff.Count > 0 || checkpointsOn.Count > 0) + Log.Debug("Icons: {Off} checkpoints-off, {On} checkpoints-on", checkpointsOff.Count, checkpointsOn.Count); + } + // Gray for correlation tracking (player zeroed) var grayForCorr = new Mat(); Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY); @@ -159,7 +200,9 @@ public class MinimapCapture : IFrameConsumer, IDisposable WallMask: wallMask, ClassifiedMat: classified, PlayerOffset: playerOffset, - Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() + Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + CheckpointsOff: checkpointsOff, + CheckpointsOn: checkpointsOn ); } @@ -374,5 +417,6 @@ public class MinimapCapture : IFrameConsumer, IDisposable public void Dispose() { _lastFrame?.Dispose(); + _iconDetector?.Dispose(); } } diff --git a/src/Poe2Trade.Navigation/NavigationExecutor.cs b/src/Poe2Trade.Navigation/NavigationExecutor.cs index 8a88355..cff63d4 100644 --- a/src/Poe2Trade.Navigation/NavigationExecutor.cs +++ b/src/Poe2Trade.Navigation/NavigationExecutor.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using OpenCvSharp; using Poe2Trade.Core; using Poe2Trade.Game; using Poe2Trade.Screen; @@ -25,15 +26,23 @@ public class NavigationExecutor : IDisposable private double _desiredDirX, _desiredDirY; private bool _directionChanged; + // Path following state (only touched by capture loop thread) + private List? _currentPath; + private int _pathIndex; + private long _lastPathTime; + + // Checkpoint collection + private Point? _checkpointGoal; + // Valid state transitions (from → allowed targets) private static readonly Dictionary ValidTransitions = new() { [NavigationState.Idle] = [NavigationState.Capturing], [NavigationState.Capturing] = [NavigationState.Processing, NavigationState.Idle], - [NavigationState.Processing] = [NavigationState.Planning, NavigationState.Stuck, NavigationState.Capturing, NavigationState.Failed, NavigationState.Idle], + [NavigationState.Processing] = [NavigationState.Planning, NavigationState.Moving, NavigationState.Stuck, NavigationState.Capturing, NavigationState.Failed, NavigationState.Idle], [NavigationState.Planning] = [NavigationState.Moving, NavigationState.Completed, NavigationState.Idle], [NavigationState.Moving] = [NavigationState.Capturing, NavigationState.Idle], - [NavigationState.Stuck] = [NavigationState.Moving, NavigationState.Completed, NavigationState.Idle], + [NavigationState.Stuck] = [NavigationState.Planning, NavigationState.Moving, NavigationState.Completed, NavigationState.Idle], [NavigationState.Completed] = [NavigationState.Idle], [NavigationState.Failed] = [NavigationState.Capturing, NavigationState.Idle], }; @@ -92,6 +101,10 @@ public class NavigationExecutor : IDisposable _capture.ResetAdaptation(); _stopped = false; _stuck.Reset(); + _currentPath = null; + _pathIndex = 0; + _lastPathTime = 0; + _checkpointGoal = null; SetState(NavigationState.Idle); Log.Information("Navigation reset (new area)"); } @@ -103,7 +116,6 @@ public class NavigationExecutor : IDisposable Log.Information("Starting explore loop"); _cachedViewport = _worldMap.GetViewportSnapshot(_worldMap.Position); - var lastMoveTime = 0L; // Input loop runs concurrently — handles WASD keys + combat clicks var inputTask = RunInputLoop(); @@ -128,7 +140,8 @@ public class NavigationExecutor : IDisposable SetState(NavigationState.Processing); var mode = _capture.DetectedMode; - var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, mode); + var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, mode, + frame.CheckpointsOff, frame.CheckpointsOn); if (_worldMap.LastMatchSucceeded) { _capture.CommitWallColors(); @@ -140,40 +153,87 @@ public class NavigationExecutor : IDisposable // Stuck detection _stuck.Update(pos); - // 2. Movement decisions — non-blocking, just post direction to input loop - var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - var moveInterval = _stuck.IsStuck ? 200 : _config.MovementWaitMs; - - if (now - lastMoveTime >= moveInterval) + // 2. Checkpoint goal check: if we're near the goal, clear it + if (_checkpointGoal is { } goal) { - lastMoveTime = now; + var gdx = goal.X - pos.X; + var gdy = goal.Y - pos.Y; + if (gdx * gdx + gdy * gdy < 20 * 20) + { + Log.Information("Checkpoint reached at ({X},{Y})", goal.X, goal.Y); + _checkpointGoal = null; + _currentPath = null; // force re-path + } + } + // 3. Re-path when needed (path consumed, stuck, or stale) + var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + if (NeedsRepath(now)) + { if (_stuck.IsStuck) SetState(NavigationState.Stuck); else SetState(NavigationState.Planning); - // BFS finds path through explored cells to nearest frontier - var direction = _worldMap.FindNearestUnexplored(pos); + (double dirX, double dirY)? direction = null; + if (_checkpointGoal is { } cpGoal) + { + // Try to path to checkpoint goal + direction = _worldMap.FindPathToTarget(pos, cpGoal); + if (direction == null) + { + Log.Information("Checkpoint unreachable, clearing goal"); + _checkpointGoal = null; + } + } + + if (_checkpointGoal == null) + { + // Look for nearby checkpoint to collect opportunistically + var nearCp = _worldMap.GetNearestCheckpointOff(pos); + if (nearCp != null) + { + _checkpointGoal = nearCp.Value; + direction = _worldMap.FindPathToTarget(pos, nearCp.Value); + if (direction != null) + Log.Information("Detouring to checkpoint at ({X},{Y})", nearCp.Value.X, nearCp.Value.Y); + else + _checkpointGoal = null; // unreachable, fall through to frontier + } + } + + // Fall back to normal frontier exploration if (direction == null) { - Log.Information("Map fully explored"); - SetState(NavigationState.Completed); - break; + direction = _worldMap.FindNearestUnexplored(pos); + if (direction == null) + { + Log.Information("Map fully explored"); + SetState(NavigationState.Completed); + break; + } } - SetState(NavigationState.Moving); - // Post direction to input loop (thread-safe) - lock (_dirLock) - { - _desiredDirX = direction.Value.dirX; - _desiredDirY = direction.Value.dirY; - _directionChanged = true; - } + _currentPath = _worldMap.LastBfsPath; + _pathIndex = 0; + _lastPathTime = now; if (_stuck.IsStuck) - _stuck.Reset(); // reset after re-routing + _stuck.Reset(); + } + + // 3. Every frame: follow path waypoints → post direction to input loop + var dir = FollowPath(pos); + if (dir != null) + { + SetState(NavigationState.Moving); + lock (_dirLock) + { + _desiredDirX = dir.Value.dirX; + _desiredDirY = dir.Value.dirY; + _directionChanged = true; + } } } catch (Exception ex) @@ -304,6 +364,61 @@ public class NavigationExecutor : IDisposable } } + /// + /// Whether a new BFS path is needed: path consumed, stuck, or stale (>3s). + /// + private bool NeedsRepath(long nowMs) + { + if (_currentPath == null || _pathIndex >= _currentPath.Count) return true; + if (_stuck.IsStuck) return true; + if (nowMs - _lastPathTime >= 3000) return true; + return false; + } + + /// + /// Follow the current BFS path: advance past passed waypoints, then compute + /// direction toward a look-ahead point ~30px ahead on the path. + /// + private (double dirX, double dirY)? FollowPath(MapPosition pos) + { + if (_currentPath == null || _pathIndex >= _currentPath.Count) + return null; + + var px = pos.X; + var py = pos.Y; + + // Advance past waypoints the player has already passed (within 15px) + while (_pathIndex < _currentPath.Count - 1) + { + var wp = _currentPath[_pathIndex]; + var dx = wp.X - px; + var dy = wp.Y - py; + if (dx * dx + dy * dy > 15 * 15) break; + _pathIndex++; + } + + // Find look-ahead target: walk forward until >= 30px from player + var target = _currentPath[^1]; // default to path end + for (var i = _pathIndex; i < _currentPath.Count; i++) + { + var wp = _currentPath[i]; + var dx = wp.X - px; + var dy = wp.Y - py; + if (dx * dx + dy * dy >= 30 * 30) + { + target = wp; + break; + } + } + + var tdx = target.X - px; + var tdy = target.Y - py; + var len = Math.Sqrt(tdx * tdx + tdy * tdy); + if (len < 1) return null; // essentially at target + + return (tdx / len, tdy / len); + } + /// /// Pick the best enemy target from detection snapshot. /// Prefers health-bar-confirmed enemies, then closest to screen center. @@ -385,7 +500,8 @@ public class NavigationExecutor : IDisposable } var stitchStart = sw.Elapsed.TotalMilliseconds; - var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, _capture.DetectedMode); + var pos = _worldMap.MatchAndStitch(frame.ClassifiedMat, frame.WallMask, _capture.DetectedMode, + frame.CheckpointsOff, frame.CheckpointsOn); if (_worldMap.LastMatchSucceeded) _capture.CommitWallColors(); var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart; diff --git a/src/Poe2Trade.Navigation/NavigationTypes.cs b/src/Poe2Trade.Navigation/NavigationTypes.cs index 02ca91e..2d05dd2 100644 --- a/src/Poe2Trade.Navigation/NavigationTypes.cs +++ b/src/Poe2Trade.Navigation/NavigationTypes.cs @@ -49,7 +49,9 @@ public record MinimapFrame( Mat WallMask, Mat ClassifiedMat, Point2d PlayerOffset, - long Timestamp + long Timestamp, + List? CheckpointsOff = null, + List? CheckpointsOn = null ) : IDisposable { public void Dispose() diff --git a/src/Poe2Trade.Navigation/PathFinder.cs b/src/Poe2Trade.Navigation/PathFinder.cs index 77b623f..df6f372 100644 --- a/src/Poe2Trade.Navigation/PathFinder.cs +++ b/src/Poe2Trade.Navigation/PathFinder.cs @@ -7,6 +7,7 @@ namespace Poe2Trade.Navigation; /// Last BFS result for visualization. /// internal record BfsResult( + List Path, // canvas coords: player → target frontier (subsampled) List BestFrontier, // frontier cells in the chosen direction (canvas coords) List OtherFrontier, // frontier cells in other directions (canvas coords) double DirX, double DirY, // chosen direction (unit vector) @@ -22,10 +23,11 @@ internal class PathFinder public BfsResult? LastResult { get; private set; } /// - /// BFS through walkable (Explored) cells to find the best frontier direction. - /// Instead of stopping at the nearest frontier, runs the full BFS and counts - /// frontier cells reachable per first-step direction. Prefers directions with - /// more frontier cells (corridors) over directions with few (dead-end rooms). + /// BFS through walkable (Explored) cells to find the best frontier cluster. + /// Runs full BFS, collects frontier cells (explored cells bordering unknown), + /// groups them into connected components, then picks the cluster with the + /// highest score = gain / (cost + 1) where gain = cluster size and cost = BFS + /// distance to the nearest cell in the cluster. /// public (double dirX, double dirY)? FindNearestUnexplored(Mat canvas, int canvasSize, MapPosition pos, int searchRadius = 400) { @@ -37,39 +39,42 @@ internal class PathFinder var size = canvasSize; var rr = searchRadius / step; var gridW = 2 * rr + 1; + var gridLen = gridW * gridW; - var visited = new bool[gridW * gridW]; - // Propagate first step from start during BFS (avoids per-frontier trace-back) - var firstStepX = new short[gridW * gridW]; - var firstStepY = new short[gridW * gridW]; + var visited = new bool[gridLen]; + var dist = new short[gridLen]; + var parentX = new short[gridLen]; + var parentY = new short[gridLen]; var queue = new Queue<(int gx, int gy)>(4096); var startGx = rr; var startGy = rr; - visited[startGy * gridW + startGx] = true; + var startIdx = startGy * gridW + startGx; + visited[startIdx] = true; + parentX[startIdx] = (short)startGx; + parentY[startIdx] = (short)startGy; queue.Enqueue((startGx, startGy)); // 8-connected neighbors ReadOnlySpan dxs = [-1, 0, 1, -1, 1, -1, 0, 1]; ReadOnlySpan dys = [-1, -1, -1, 0, 0, 1, 1, 1]; - // Count frontier cells per first-step direction, and collect positions - var frontierCounts = new Dictionary(); - var firstStepCoords = new Dictionary(); - var frontierByKey = new Dictionary>(); + // Step A: BFS flood-fill, recording dist and parents + var isFrontier = new bool[gridLen]; + var frontierCells = new List<(int gx, int gy)>(); while (queue.Count > 0) { var (gx, gy) = queue.Dequeue(); + var cellIdx = gy * gridW + gx; // Map grid coords back to canvas coords var wx = cx + (gx - rr) * step; var wy = cy + (gy - rr) * step; - // Check if this explored cell borders Unknown/Fog (= frontier) + // Step B: Check if this explored cell borders Unknown/Fog (= frontier) if (gx != startGx || gy != startGy) { - var cellIdx = gy * gridW + gx; for (var d = 0; d < 8; d++) { var nx = wx + dxs[d] * step; @@ -78,17 +83,9 @@ internal class PathFinder var neighbor = canvas.At(ny, nx); if (neighbor == (byte)MapCell.Unknown || neighbor == (byte)MapCell.Fog) { - var fsKey = firstStepY[cellIdx] * gridW + firstStepX[cellIdx]; - if (frontierCounts.TryGetValue(fsKey, out var cnt)) - frontierCounts[fsKey] = cnt + 1; - else - { - frontierCounts[fsKey] = 1; - firstStepCoords[fsKey] = (firstStepX[cellIdx], firstStepY[cellIdx]); - frontierByKey[fsKey] = []; - } - frontierByKey[fsKey].Add(new Point(wx, wy)); - break; // don't double-count this cell + isFrontier[cellIdx] = true; + frontierCells.Add((gx, gy)); + break; } } } @@ -108,65 +105,264 @@ internal class PathFinder if (nwx < 0 || nwx >= size || nwy < 0 || nwy >= size) continue; var cell = canvas.At(nwy, nwx); - if (cell != (byte)MapCell.Explored) continue; + if (cell != (byte)MapCell.Explored && cell != (byte)MapCell.Fog) continue; visited[idx] = true; - // Propagate first step: direct neighbors of start ARE the first step - if (gx == startGx && gy == startGy) - { - firstStepX[idx] = (short)ngx; - firstStepY[idx] = (short)ngy; - } - else - { - var parentIdx = gy * gridW + gx; - firstStepX[idx] = firstStepX[parentIdx]; - firstStepY[idx] = firstStepY[parentIdx]; - } + dist[idx] = (short)(dist[cellIdx] + 1); + parentX[idx] = (short)gx; + parentY[idx] = (short)gy; queue.Enqueue((ngx, ngy)); } } - if (frontierCounts.Count == 0) + if (frontierCells.Count == 0) { Log.Information("BFS: no reachable frontier within {Radius}px", searchRadius); LastResult = null; return null; } - // Pick direction with the most frontier cells (prefers corridors over dead ends) - var bestKey = -1; - var bestCount = 0; - foreach (var (key, count) in frontierCounts) + // Step C: Cluster frontiers via 8-connected flood-fill + var clusterVisited = new bool[gridLen]; + var clusterCount = 0; + var bestClusterGain = 0; + var bestClusterCost = 0; + var bestClusterScore = -1.0; + var bestEntryGx = -1; + var bestEntryGy = -1; + List? bestFrontierPts = null; + var otherFrontier = new List(); + var clusterQueue = new Queue<(int gx, int gy)>(256); + + foreach (var (fgx, fgy) in frontierCells) { - if (count > bestCount) + var fIdx = fgy * gridW + fgx; + if (clusterVisited[fIdx]) continue; + + clusterCount++; + // Flood-fill this cluster + var clusterCells = new List<(int gx, int gy)>(); + var minDist = dist[fIdx]; + var entryGx = fgx; + var entryGy = fgy; + + clusterVisited[fIdx] = true; + clusterQueue.Enqueue((fgx, fgy)); + + while (clusterQueue.Count > 0) { - bestCount = count; - bestKey = key; + var (cgx, cgy) = clusterQueue.Dequeue(); + clusterCells.Add((cgx, cgy)); + + var cIdx = cgy * gridW + cgx; + if (dist[cIdx] < minDist) + { + minDist = dist[cIdx]; + entryGx = cgx; + entryGy = cgy; + } + + for (var d = 0; d < 8; d++) + { + var ngx = cgx + dxs[d]; + var ngy = cgy + dys[d]; + if (ngx < 0 || ngx >= gridW || ngy < 0 || ngy >= gridW) continue; + var nIdx = ngy * gridW + ngx; + if (clusterVisited[nIdx] || !isFrontier[nIdx]) continue; + clusterVisited[nIdx] = true; + clusterQueue.Enqueue((ngx, ngy)); + } + } + + // Step D: Score this cluster + var gain = clusterCells.Count; + var cost = (int)minDist; + var score = gain / (cost + 1.0); + + // Convert cluster cells to canvas coords + var pts = new List(gain); + foreach (var (cgx, cgy) in clusterCells) + pts.Add(new Point(cx + (cgx - rr) * step, cy + (cgy - rr) * step)); + + if (score > bestClusterScore) + { + // Demote previous best to "other" + if (bestFrontierPts != null) + otherFrontier.AddRange(bestFrontierPts); + + bestClusterScore = score; + bestClusterGain = gain; + bestClusterCost = cost; + bestEntryGx = entryGx; + bestEntryGy = entryGy; + bestFrontierPts = pts; + } + else + { + otherFrontier.AddRange(pts); } } - var (bestGx, bestGy) = firstStepCoords[bestKey]; - var dirX = (double)(bestGx - startGx); - var dirY = (double)(bestGy - startGy); - var len = Math.Sqrt(dirX * dirX + dirY * dirY); - if (len < 0.001) return (1, 0); + // Step E: Trace path from entry cell of winning cluster back to start + var rawPath = new List(); + var traceGx = bestEntryGx; + var traceGy = bestEntryGy; + while (traceGx != startGx || traceGy != startGy) + { + var wx = cx + (traceGx - rr) * step; + var wy = cy + (traceGy - rr) * step; + rawPath.Add(new Point(wx, wy)); + var tIdx = traceGy * gridW + traceGx; + var pgx = parentX[tIdx]; + var pgy = parentY[tIdx]; + traceGx = pgx; + traceGy = pgy; + } + rawPath.Add(new Point(cx, cy)); + rawPath.Reverse(); // player → frontier + // Subsample: every 4th point (~8px apart on canvas with step=2) + var path = new List(); + for (var i = 0; i < rawPath.Count; i += 4) + path.Add(rawPath[i]); + if (path.Count == 0 || path[^1] != rawPath[^1]) + path.Add(rawPath[^1]); // always include endpoint + + // Direction from start toward first path segment + var dirX = (double)(path[Math.Min(1, path.Count - 1)].X - cx); + var dirY = (double)(path[Math.Min(1, path.Count - 1)].Y - cy); + var len = Math.Sqrt(dirX * dirX + dirY * dirY); + if (len < 0.001) { dirX = 1; dirY = 0; len = 1; } dirX /= len; dirY /= len; - // Store result for visualization - var bestFrontier = frontierByKey[bestKey]; - var otherFrontier = new List(); - foreach (var (key, pts) in frontierByKey) - { - if (key != bestKey) - otherFrontier.AddRange(pts); - } - LastResult = new BfsResult(bestFrontier, otherFrontier, dirX, dirY, cx, cy); + // Step F: Store result for visualization + LastResult = new BfsResult(path, bestFrontierPts!, otherFrontier, dirX, dirY, cx, cy); - Log.Debug("BFS: {DirCount} directions, best={Best} frontier cells, dir=({Dx:F2},{Dy:F2})", - frontierCounts.Count, bestCount, dirX, dirY); + Log.Debug("BFS: {ClusterCount} clusters, best={Gain} cells cost={Cost} score={Score:F1}, dir=({Dx:F2},{Dy:F2}), path={PathLen} waypoints", + clusterCount, bestClusterGain, bestClusterCost, bestClusterScore, dirX, dirY, path.Count); + return (dirX, dirY); + } + + /// + /// BFS to a specific target point (e.g. checkpoint). Same half-res grid as FindNearestUnexplored. + /// Terminates when a cell within ~10px of the target is reached. + /// Returns direction toward the target, or null if unreachable. + /// + public (double dirX, double dirY)? FindPathToTarget(Mat canvas, int canvasSize, MapPosition pos, Point target, int searchRadius = 400) + { + var cx = (int)Math.Round(pos.X); + var cy = (int)Math.Round(pos.Y); + + const int step = 2; + var rr = searchRadius / step; + var gridW = 2 * rr + 1; + + var visited = new bool[gridW * gridW]; + var parentX = new short[gridW * gridW]; + var parentY = new short[gridW * gridW]; + + var queue = new Queue<(int gx, int gy)>(4096); + var startGx = rr; + var startGy = rr; + var startIdx = startGy * gridW + startGx; + visited[startIdx] = true; + parentX[startIdx] = (short)startGx; + parentY[startIdx] = (short)startGy; + queue.Enqueue((startGx, startGy)); + + ReadOnlySpan dxs = [-1, 0, 1, -1, 1, -1, 0, 1]; + ReadOnlySpan dys = [-1, -1, -1, 0, 0, 1, 1, 1]; + + const int arrivalDist = 10; + const int arrivalDist2 = arrivalDist * arrivalDist; + + var foundGx = -1; + var foundGy = -1; + + while (queue.Count > 0) + { + var (gx, gy) = queue.Dequeue(); + var wx = cx + (gx - rr) * step; + var wy = cy + (gy - rr) * step; + + // Check if we've reached the target + var dtx = wx - target.X; + var dty = wy - target.Y; + if (dtx * dtx + dty * dty <= arrivalDist2) + { + foundGx = gx; + foundGy = gy; + break; + } + + for (var d = 0; d < 8; d++) + { + var ngx = gx + dxs[d]; + var ngy = gy + dys[d]; + if (ngx < 0 || ngx >= gridW || ngy < 0 || ngy >= gridW) continue; + + var idx = ngy * gridW + ngx; + if (visited[idx]) continue; + + var nwx = cx + (ngx - rr) * step; + var nwy = cy + (ngy - rr) * step; + if (nwx < 0 || nwx >= canvasSize || nwy < 0 || nwy >= canvasSize) continue; + + var cell = canvas.At(nwy, nwx); + if (cell != (byte)MapCell.Explored && cell != (byte)MapCell.Fog) continue; + + visited[idx] = true; + parentX[idx] = (short)gx; + parentY[idx] = (short)gy; + queue.Enqueue((ngx, ngy)); + } + } + + if (foundGx < 0) + { + Log.Debug("BFS target: unreachable target=({TX},{TY}) from ({CX},{CY})", target.X, target.Y, cx, cy); + return null; + } + + // Trace path back to start + var rawPath = new List(); + var traceGx = foundGx; + var traceGy = foundGy; + while (traceGx != startGx || traceGy != startGy) + { + var wx = cx + (traceGx - rr) * step; + var wy = cy + (traceGy - rr) * step; + rawPath.Add(new Point(wx, wy)); + var tIdx = traceGy * gridW + traceGx; + var pgx = parentX[tIdx]; + var pgy = parentY[tIdx]; + traceGx = pgx; + traceGy = pgy; + } + rawPath.Add(new Point(cx, cy)); + rawPath.Reverse(); + + // Subsample every 4th point + var path = new List(); + for (var i = 0; i < rawPath.Count; i += 4) + path.Add(rawPath[i]); + if (path.Count == 0 || path[^1] != rawPath[^1]) + path.Add(rawPath[^1]); + + // Direction from start toward first path segment + var dirX = (double)(path[Math.Min(1, path.Count - 1)].X - cx); + var dirY = (double)(path[Math.Min(1, path.Count - 1)].Y - cy); + var len = Math.Sqrt(dirX * dirX + dirY * dirY); + if (len < 0.001) { dirX = 1; dirY = 0; len = 1; } + dirX /= len; + dirY /= len; + + // Store result for visualization (reuse BfsResult — frontiers empty for target mode) + LastResult = new BfsResult(path, [], [], dirX, dirY, cx, cy); + + Log.Debug("BFS target: path={PathLen} waypoints to ({TX},{TY}), dir=({Dx:F2},{Dy:F2})", + path.Count, target.X, target.Y, dirX, dirY); return (dirX, dirY); } } diff --git a/src/Poe2Trade.Navigation/WorldMap.cs b/src/Poe2Trade.Navigation/WorldMap.cs index 3d545c5..965b755 100644 --- a/src/Poe2Trade.Navigation/WorldMap.cs +++ b/src/Poe2Trade.Navigation/WorldMap.cs @@ -16,9 +16,15 @@ public class WorldMap : IDisposable private Mat? _prevWallMask; // for frame deduplication private readonly PathFinder _pathFinder = new(); + // Checkpoint tracking (canvas coordinates) + private readonly List<(Point Pos, long LastSeenMs)> _checkpointsOff = []; + private readonly List<(Point Pos, long LastSeenMs)> _checkpointsOn = []; + private const int CheckpointDedupRadius = 20; + public MapPosition Position => _position; public bool LastMatchSucceeded { get; private set; } public int CanvasSize => _canvasSize; + internal List? LastBfsPath => _pathFinder.LastResult?.Path; private const int GrowMargin = 500; private const int GrowAmount = 2000; // 1000px added per side @@ -64,7 +70,8 @@ public class WorldMap : IDisposable /// Match current wall mask against the accumulated map to find position, /// then stitch walls and paint explored area. /// - public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask, MinimapMode mode = MinimapMode.Overlay) + public MapPosition MatchAndStitch(Mat classifiedMat, Mat wallMask, MinimapMode mode = MinimapMode.Overlay, + List? checkpointsOff = null, List? checkpointsOn = null) { EnsureCapacity(); var sw = Stopwatch.StartNew(); @@ -103,6 +110,7 @@ public class WorldMap : IDisposable StitchWithConfidence(classifiedMat, _position, boosted: true, mode: mode); PaintExploredCircle(_position); + MergeCheckpoints(_position, classifiedMat.Width, checkpointsOff, checkpointsOn); LastMatchSucceeded = true; // signal caller to update viewport if (_consecutiveMatchFails >= 30) { @@ -140,6 +148,7 @@ public class WorldMap : IDisposable var stitchStart = sw.Elapsed.TotalMilliseconds; StitchWithConfidence(classifiedMat, _position, boosted: false, mode: mode); PaintExploredCircle(_position); + MergeCheckpoints(_position, classifiedMat.Width, checkpointsOff, checkpointsOn); var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart; var posDx = _position.X - prevPos.X; @@ -323,24 +332,23 @@ public class WorldMap : IDisposable dstRoi.Set(row, col, (byte)MapCell.Explored); } - // Mark fog on canvas + // Mark fog on canvas (on top of Unknown or Explored — not Wall) if (isCorner) { - // Corner minimap: fog is clean, accept all fog pixels for (var row = 0; row < h; row++) for (var col = 0; col < w; col++) { if (srcRoi.At(row, col) != (byte)MapCell.Fog) continue; - if (dstRoi.At(row, col) == (byte)MapCell.Unknown) + var dst = dstRoi.At(row, col); + if (dst == (byte)MapCell.Unknown || dst == (byte)MapCell.Explored) dstRoi.Set(row, col, (byte)MapCell.Fog); } } else { - // Overlay: restrict fog to ring around player (filters spell effect noise) - var fogInner2 = _config.ExploredRadius * _config.ExploredRadius; - var fogOuter = _config.ExploredRadius + 5; - var fogOuter2 = fogOuter * fogOuter; + // Overlay: exclude small area near player center (spell effect noise) + const int fogInner = 30; + const int fogInner2 = fogInner * fogInner; for (var row = 0; row < h; row++) for (var col = 0; col < w; col++) @@ -349,10 +357,10 @@ public class WorldMap : IDisposable var fx = srcX + col - halfSize; var fy = srcY + row - halfSize; - var dist2 = fx * fx + fy * fy; - if (dist2 < fogInner2 || dist2 > fogOuter2) continue; + if (fx * fx + fy * fy < fogInner2) continue; - if (dstRoi.At(row, col) == (byte)MapCell.Unknown) + var dst = dstRoi.At(row, col); + if (dst == (byte)MapCell.Unknown || dst == (byte)MapCell.Explored) dstRoi.Set(row, col, (byte)MapCell.Fog); } } @@ -360,8 +368,10 @@ public class WorldMap : IDisposable } /// - /// Mark explored area: circle around player position, overwrite Unknown and Fog cells. - /// Called by MatchAndStitch after stitch in both bootstrap and match-success branches. + /// Mark explored area: circle around player position. + /// Unknown cells are marked Explored within the full radius. + /// Fog cells are only cleared in the inner portion (fogClearRadius), + /// preserving fog at the edge so it stays visible on the viewport. /// private void PaintExploredCircle(MapPosition position) { @@ -369,6 +379,8 @@ public class WorldMap : IDisposable var pcy = (int)Math.Round(position.Y); var r = _config.ExploredRadius; var r2 = r * r; + var fogClear = r - 20; + var fogClear2 = fogClear * fogClear; var y0 = Math.Max(0, pcy - r); var y1 = Math.Min(_canvasSize - 1, pcy + r); @@ -380,9 +392,12 @@ public class WorldMap : IDisposable { var dx = x - pcx; var dy = y - pcy; - if (dx * dx + dy * dy > r2) continue; + var d2 = dx * dx + dy * dy; + if (d2 > r2) continue; var cell = _canvas.At(y, x); - if (cell == (byte)MapCell.Unknown || cell == (byte)MapCell.Fog) + if (cell == (byte)MapCell.Unknown) + _canvas.Set(y, x, (byte)MapCell.Explored); + else if (cell == (byte)MapCell.Fog && d2 <= fogClear2) _canvas.Set(y, x, (byte)MapCell.Explored); } } @@ -430,6 +445,9 @@ public class WorldMap : IDisposable public (double dirX, double dirY)? FindNearestUnexplored(MapPosition pos, int searchRadius = 400) => _pathFinder.FindNearestUnexplored(_canvas, _canvasSize, pos, searchRadius); + public (double dirX, double dirY)? FindPathToTarget(MapPosition pos, Point target, int searchRadius = 400) + => _pathFinder.FindPathToTarget(_canvas, _canvasSize, pos, target, searchRadius); + public byte[] GetMapSnapshot() { Cv2.ImEncode(".png", _canvas, out var buf); @@ -456,7 +474,7 @@ public class WorldMap : IDisposable else if (v == (byte)MapCell.Wall) colored.Set(r, c, new Vec3b(26, 45, 61)); else if (v == (byte)MapCell.Fog) - colored.Set(r, c, new Vec3b(55, 40, 28)); + colored.Set(r, c, new Vec3b(120, 70, 40)); } // BFS overlay: frontier cells + direction line @@ -489,6 +507,32 @@ public class WorldMap : IDisposable var ey = (int)(py2 + bfs.DirY * lineLen); Cv2.ArrowedLine(colored, new Point(px2, py2), new Point(ex, ey), new Scalar(0, 220, 0), 2, tipLength: 0.3); + + // BFS path polyline (bright green) + if (bfs.Path.Count >= 2) + { + var viewPath = new Point[bfs.Path.Count]; + for (var i = 0; i < bfs.Path.Count; i++) + viewPath[i] = new Point(bfs.Path[i].X - x0, bfs.Path[i].Y - y0); + Cv2.Polylines(colored, [viewPath], isClosed: false, + color: new Scalar(0, 255, 0), thickness: 1); + } + } + + // Checkpoint markers + foreach (var (cp, _) in _checkpointsOff) + { + var cpx = cp.X - x0; + var cpy = cp.Y - y0; + if (cpx >= 0 && cpx < viewSize && cpy >= 0 && cpy < viewSize) + Cv2.Circle(colored, new Point(cpx, cpy), 5, new Scalar(100, 100, 100), -1); // gray + } + foreach (var (cp, _) in _checkpointsOn) + { + var cpx = cp.X - x0; + var cpy = cp.Y - y0; + if (cpx >= 0 && cpx < viewSize && cpy >= 0 && cpy < viewSize) + Cv2.Circle(colored, new Point(cpx, cpy), 5, new Scalar(220, 200, 0), -1); // bright cyan (BGR) } // Player dot (orange, on top) @@ -501,6 +545,90 @@ public class WorldMap : IDisposable return buf; } + /// + /// Convert frame-relative checkpoint positions to canvas coords and merge with existing lists. + /// When a checkpoint-on appears near a checkpoint-off, remove the off entry. + /// + private void MergeCheckpoints(MapPosition position, int frameSize, + List? offPoints, List? onPoints) + { + var halfSize = frameSize / 2; + var canvasX = (int)Math.Round(position.X) - halfSize; + var canvasY = (int)Math.Round(position.Y) - halfSize; + var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var r2 = CheckpointDedupRadius * CheckpointDedupRadius; + + if (offPoints != null) + { + foreach (var fp in offPoints) + { + var cp = new Point(canvasX + fp.X, canvasY + fp.Y); + if (!IsDuplicate(_checkpointsOff, cp, r2)) + _checkpointsOff.Add((cp, now)); + } + } + + if (onPoints != null) + { + foreach (var fp in onPoints) + { + var cp = new Point(canvasX + fp.X, canvasY + fp.Y); + + // Remove matching off-checkpoint (it got activated) + for (var i = _checkpointsOff.Count - 1; i >= 0; i--) + { + var dx = _checkpointsOff[i].Pos.X - cp.X; + var dy = _checkpointsOff[i].Pos.Y - cp.Y; + if (dx * dx + dy * dy <= r2) + { + _checkpointsOff.RemoveAt(i); + break; + } + } + + if (!IsDuplicate(_checkpointsOn, cp, r2)) + _checkpointsOn.Add((cp, now)); + } + } + } + + private static bool IsDuplicate(List<(Point Pos, long LastSeenMs)> list, Point pt, int r2) + { + foreach (var (pos, _) in list) + { + var dx = pos.X - pt.X; + var dy = pos.Y - pt.Y; + if (dx * dx + dy * dy <= r2) + return true; + } + return false; + } + + /// + /// Returns the nearest unactivated checkpoint within maxDist canvas pixels, or null. + /// + public Point? GetNearestCheckpointOff(MapPosition pos, int maxDist = 200) + { + var px = (int)Math.Round(pos.X); + var py = (int)Math.Round(pos.Y); + var maxDist2 = maxDist * maxDist; + + Point? best = null; + var bestDist2 = int.MaxValue; + foreach (var (cp, _) in _checkpointsOff) + { + var dx = cp.X - px; + var dy = cp.Y - py; + var d2 = dx * dx + dy * dy; + if (d2 <= maxDist2 && d2 < bestDist2) + { + bestDist2 = d2; + best = cp; + } + } + return best; + } + public void Reset() { _canvas.Dispose(); @@ -514,6 +642,8 @@ public class WorldMap : IDisposable _frameCount = 0; _consecutiveMatchFails = 0; LastMatchSucceeded = false; + _checkpointsOff.Clear(); + _checkpointsOn.Clear(); } ///