started adding navigation

src/Poe2Trade.Navigation/MinimapCapture.cs

@@ -0,0 +1,119 @@
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.Runtime.InteropServices;
+using OpenCvSharp;
+using OpenCvSharp.Extensions;
+using Serilog;
+using Region = Poe2Trade.Core.Region;
+using Point = OpenCvSharp.Point;
+using Size = OpenCvSharp.Size;
+
+namespace Poe2Trade.Navigation;
+
+public class MinimapCapture : IDisposable
+{
+    private readonly MinimapConfig _config;
+    private Mat? _circularMask;
+
+    [DllImport("user32.dll")]
+    private static extern int GetSystemMetrics(int nIndex);
+
+    public MinimapCapture(MinimapConfig config)
+    {
+        _config = config;
+        BuildCircularMask();
+    }
+
+    private void BuildCircularMask()
+    {
+        var size = _config.CaptureSize;
+        _circularMask = new Mat(size, size, MatType.CV_8UC1, Scalar.Black);
+        var center = new Point(size / 2, size / 2);
+        Cv2.Circle(_circularMask, center, _config.FogRadius, Scalar.White, -1);
+    }
+
+    public MinimapFrame? CaptureFrame()
+    {
+        var region = _config.CaptureRegion;
+        using var bitmap = CaptureScreen(region);
+        using var bgr = BitmapConverter.ToMat(bitmap);
+
+        if (bgr.Empty())
+            return null;
+
+        // Apply circular mask to ignore area outside fog-of-war circle
+        using var masked = new Mat();
+        Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
+
+        // Convert to HSV
+        using var hsv = new Mat();
+        Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
+
+        // Classify explored areas
+        using var exploredMask = new Mat();
+        Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
+
+        // Classify walls: dark pixels within the circular mask
+        using var valueChan = new Mat();
+        Cv2.ExtractChannel(hsv, valueChan, 2); // V channel
+        using var darkMask = new Mat();
+        Cv2.Threshold(valueChan, darkMask, _config.WallMaxValue, 255, ThresholdTypes.BinaryInv);
+        // Apply morphological close to connect wall fragments
+        using var wallKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
+        using var wallMask = new Mat();
+        Cv2.MorphologyEx(darkMask, wallMask, MorphTypes.Close, wallKernel);
+        // Only within circular mask
+        Cv2.BitwiseAnd(wallMask, _circularMask!, wallMask);
+
+        // Build classified mat: Unknown=0, Explored=1, Wall=2
+        var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
+        classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
+        classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
+        // Ensure only within circular mask
+        Cv2.BitwiseAnd(classified, _circularMask!, classified);
+
+        // Detect player marker (orange X)
+        using var playerMask = new Mat();
+        Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
+        var playerOffset = FindCentroid(playerMask);
+
+        // Convert to grayscale for phase correlation
+        var gray = new Mat();
+        Cv2.CvtColor(masked, gray, ColorConversionCodes.BGR2GRAY);
+        // Apply circular mask to gray too
+        Cv2.BitwiseAnd(gray, _circularMask!, gray);
+
+        return new MinimapFrame(
+            GrayMat: gray,
+            ClassifiedMat: classified,
+            PlayerOffset: playerOffset,
+            Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
+        );
+    }
+
+    private Point2d FindCentroid(Mat mask)
+    {
+        var moments = Cv2.Moments(mask, true);
+        if (moments.M00 < 10) // not enough pixels
+            return new Point2d(0, 0);
+
+        var cx = moments.M10 / moments.M00 - _config.CaptureSize / 2.0;
+        var cy = moments.M01 / moments.M00 - _config.CaptureSize / 2.0;
+        return new Point2d(cx, cy);
+    }
+
+    private static Bitmap CaptureScreen(Region region)
+    {
+        var bitmap = new Bitmap(region.Width, region.Height, PixelFormat.Format32bppArgb);
+        using var g = Graphics.FromImage(bitmap);
+        g.CopyFromScreen(region.X, region.Y, 0, 0,
+            new System.Drawing.Size(region.Width, region.Height),
+            CopyPixelOperation.SourceCopy);
+        return bitmap;
+    }
+
+    public void Dispose()
+    {
+        _circularMask?.Dispose();
+    }
+}

src/Poe2Trade.Navigation/NavigationExecutor.cs

@@ -0,0 +1,156 @@
+using Poe2Trade.Core;
+using Poe2Trade.Game;
+using Serilog;
+
+namespace Poe2Trade.Navigation;
+
+public class NavigationExecutor : IDisposable
+{
+    private readonly IGameController _game;
+    private readonly MinimapConfig _config;
+    private readonly MinimapCapture _capture;
+    private readonly PositionTracker _tracker;
+    private readonly WorldMap _worldMap;
+    private NavigationState _state = NavigationState.Idle;
+    private bool _stopped;
+    private static readonly Random Rng = new();
+
+    public event Action<NavigationState>? StateChanged;
+    public NavigationState State => _state;
+
+    public NavigationExecutor(IGameController game, MinimapConfig? config = null)
+    {
+        _game = game;
+        _config = config ?? new MinimapConfig();
+        _capture = new MinimapCapture(_config);
+        _tracker = new PositionTracker(_config);
+        _worldMap = new WorldMap(_config);
+    }
+
+    private void SetState(NavigationState s)
+    {
+        _state = s;
+        StateChanged?.Invoke(s);
+    }
+
+    public Task Stop()
+    {
+        _stopped = true;
+        SetState(NavigationState.Idle);
+        Log.Information("Navigation executor stopped");
+        return Task.CompletedTask;
+    }
+
+    public void Reset()
+    {
+        _tracker.Reset();
+        _worldMap.Reset();
+        _stopped = false;
+        SetState(NavigationState.Idle);
+    }
+
+    public async Task RunExploreLoop()
+    {
+        _stopped = false;
+        Log.Information("Starting explore loop");
+
+        // Open minimap overlay (Tab)
+        await _game.ToggleMinimap();
+        await Helpers.Sleep(300);
+
+        while (!_stopped)
+        {
+            try
+            {
+                // 1. Capture frame
+                SetState(NavigationState.Capturing);
+                using var frame = _capture.CaptureFrame();
+                if (frame == null)
+                {
+                    Log.Warning("Failed to capture minimap frame");
+                    await Helpers.Sleep(200);
+                    continue;
+                }
+
+                // 2. Track position via phase correlation
+                SetState(NavigationState.Processing);
+                var pos = _tracker.UpdatePosition(frame.GrayMat);
+
+                // 3. Stitch into world map
+                _worldMap.StitchFrame(frame.ClassifiedMat, pos);
+
+                // 4. Check if stuck
+                if (_tracker.IsStuck)
+                {
+                    SetState(NavigationState.Stuck);
+                    Log.Information("Stuck detected, clicking random direction");
+                    await ClickRandomDirection();
+                    await Helpers.Sleep(_config.MovementWaitMs);
+                    continue;
+                }
+
+                // 5. Find best exploration direction
+                SetState(NavigationState.Planning);
+                var direction = _worldMap.FindNearestUnexplored(pos);
+
+                if (direction == null)
+                {
+                    Log.Information("Map fully explored");
+                    SetState(NavigationState.Completed);
+                    break;
+                }
+
+                // 6. Click to move in that direction
+                SetState(NavigationState.Moving);
+                await ClickToMove(direction.Value.dirX, direction.Value.dirY);
+
+                // 7. Wait for character to walk
+                await Helpers.Sleep(_config.MovementWaitMs);
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Error in explore loop");
+                SetState(NavigationState.Failed);
+                await Helpers.Sleep(1000);
+            }
+        }
+
+        if (_state != NavigationState.Completed)
+            SetState(NavigationState.Idle);
+
+        Log.Information("Explore loop ended");
+    }
+
+    private async Task ClickToMove(double dirX, double dirY)
+    {
+        // Player is at minimap center on screen; click offset from center
+        var len = Math.Sqrt(dirX * dirX + dirY * dirY);
+        if (len < 0.001) return;
+
+        var nx = dirX / len;
+        var ny = dirY / len;
+
+        var clickX = _config.MinimapCenterX + (int)(nx * _config.ClickRadius);
+        var clickY = _config.MinimapCenterY + (int)(ny * _config.ClickRadius);
+
+        Log.Debug("Click to move: ({X}, {Y}) dir=({Dx:F2}, {Dy:F2})", clickX, clickY, nx, ny);
+        await _game.LeftClickAt(clickX, clickY);
+    }
+
+    private async Task ClickRandomDirection()
+    {
+        var angle = Rng.NextDouble() * 2 * Math.PI;
+        await ClickToMove(Math.Cos(angle), Math.Sin(angle));
+    }
+
+    public MapPosition Position => _tracker.Position;
+    public byte[] GetMapSnapshot() => _worldMap.GetMapSnapshot();
+    public byte[] GetViewportSnapshot(int viewSize = 400) => _worldMap.GetViewportSnapshot(_tracker.Position, viewSize);
+
+    public void Dispose()
+    {
+        _capture.Dispose();
+        _tracker.Dispose();
+        _worldMap.Dispose();
+    }
+}

src/Poe2Trade.Navigation/NavigationTypes.cs

@@ -0,0 +1,82 @@
+using Poe2Trade.Core;
+using OpenCvSharp;
+
+namespace Poe2Trade.Navigation;
+
+public enum NavigationState
+{
+    Idle,
+    Capturing,
+    Processing,
+    Planning,
+    Moving,
+    Stuck,
+    Completed,
+    Failed
+}
+
+public record MapPosition(double X, double Y);
+
+public enum MapCell : byte
+{
+    Unknown = 0,
+    Explored = 1,
+    Wall = 2
+}
+
+public record MinimapFrame(
+    Mat GrayMat,
+    Mat ClassifiedMat,
+    Point2d PlayerOffset,
+    long Timestamp
+) : IDisposable
+{
+    public void Dispose()
+    {
+        GrayMat.Dispose();
+        ClassifiedMat.Dispose();
+    }
+}
+
+public class MinimapConfig
+{
+    // Minimap center on screen (2560x1440)
+    public int MinimapCenterX { get; set; } = 1280;
+    public int MinimapCenterY { get; set; } = 700;
+
+    // Capture region: 300x300 centered at minimap center
+    public Region CaptureRegion => new(
+        MinimapCenterX - CaptureSize / 2,
+        MinimapCenterY - CaptureSize / 2,
+        CaptureSize, CaptureSize);
+
+    public int CaptureSize { get; set; } = 300;
+
+    // Fog-of-war circle radius within the captured frame
+    public int FogRadius { get; set; } = 120;
+
+    // HSV range for explored areas (blue/teal)
+    public Scalar ExploredLoHSV { get; set; } = new(85, 30, 30);
+    public Scalar ExploredHiHSV { get; set; } = new(145, 255, 255);
+
+    // HSV range for player marker (orange X)
+    public Scalar PlayerLoHSV { get; set; } = new(5, 100, 100);
+    public Scalar PlayerHiHSV { get; set; } = new(25, 255, 255);
+
+    // HSV range for walls (dark pixels)
+    public int WallMaxValue { get; set; } = 40;
+
+    // Movement
+    public int ClickRadius { get; set; } = 100;
+    public int MovementWaitMs { get; set; } = 600;
+
+    // World map canvas
+    public int CanvasSize { get; set; } = 4000;
+
+    // Phase correlation confidence threshold
+    public double ConfidenceThreshold { get; set; } = 0.15;
+
+    // Stuck detection
+    public double StuckThreshold { get; set; } = 2.0;
+    public int StuckFrameCount { get; set; } = 5;
+}

src/Poe2Trade.Navigation/Poe2Trade.Navigation.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0-windows10.0.19041.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="OpenCvSharp4" Version="4.11.0.*" />
+    <PackageReference Include="OpenCvSharp4.Extensions" Version="4.11.0.*" />
+    <PackageReference Include="OpenCvSharp4.runtime.win" Version="4.11.0.*" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.12" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Poe2Trade.Core\Poe2Trade.Core.csproj" />
+    <ProjectReference Include="..\Poe2Trade.Game\Poe2Trade.Game.csproj" />
+    <ProjectReference Include="..\Poe2Trade.Screen\Poe2Trade.Screen.csproj" />
+  </ItemGroup>
+</Project>

src/Poe2Trade.Navigation/PositionTracker.cs

@@ -0,0 +1,93 @@
+using OpenCvSharp;
+using Serilog;
+
+namespace Poe2Trade.Navigation;
+
+public class PositionTracker : IDisposable
+{
+    private readonly MinimapConfig _config;
+    private Mat? _prevGray;
+    private Mat? _hanningWindow;
+    private double _worldX;
+    private double _worldY;
+    private int _stuckCounter;
+
+    public MapPosition Position => new(_worldX, _worldY);
+    public bool IsStuck => _stuckCounter >= _config.StuckFrameCount;
+
+    public PositionTracker(MinimapConfig config)
+    {
+        _config = config;
+        _worldX = config.CanvasSize / 2.0;
+        _worldY = config.CanvasSize / 2.0;
+    }
+
+    public MapPosition UpdatePosition(Mat currentGray)
+    {
+        if (_prevGray == null || _hanningWindow == null)
+        {
+            _prevGray = currentGray.Clone();
+            _hanningWindow = new Mat();
+            Cv2.CreateHanningWindow(_hanningWindow, currentGray.Size(), MatType.CV_64F);
+            return Position;
+        }
+
+        // Convert to float64 for phase correlation
+        using var prev64 = new Mat();
+        using var curr64 = new Mat();
+        _prevGray.ConvertTo(prev64, MatType.CV_64F);
+        currentGray.ConvertTo(curr64, MatType.CV_64F);
+
+        var shift = Cv2.PhaseCorrelate(prev64, curr64, _hanningWindow, out var confidence);
+
+        if (confidence < _config.ConfidenceThreshold)
+        {
+            Log.Debug("Phase correlation low confidence: {Confidence:F3}", confidence);
+            _stuckCounter++;
+            _prevGray.Dispose();
+            _prevGray = currentGray.Clone();
+            return Position;
+        }
+
+        // Negate: minimap scrolls opposite to player movement
+        var dx = -shift.X;
+        var dy = -shift.Y;
+        var displacement = Math.Sqrt(dx * dx + dy * dy);
+
+        if (displacement < _config.StuckThreshold)
+        {
+            _stuckCounter++;
+        }
+        else
+        {
+            _stuckCounter = 0;
+            _worldX += dx;
+            _worldY += dy;
+        }
+
+        Log.Debug("Position: ({X:F1}, {Y:F1}) dx={Dx:F1} dy={Dy:F1} conf={Conf:F3} stuck={Stuck}",
+            _worldX, _worldY, dx, dy, confidence, _stuckCounter);
+
+        _prevGray.Dispose();
+        _prevGray = currentGray.Clone();
+        return Position;
+    }
+
+    public void Reset()
+    {
+        _prevGray?.Dispose();
+        _prevGray = null;
+        _hanningWindow?.Dispose();
+        _hanningWindow = null;
+        _worldX = _config.CanvasSize / 2.0;
+        _worldY = _config.CanvasSize / 2.0;
+        _stuckCounter = 0;
+        Log.Information("Position tracker reset");
+    }
+
+    public void Dispose()
+    {
+        _prevGray?.Dispose();
+        _hanningWindow?.Dispose();
+    }
+}

src/Poe2Trade.Navigation/WorldMap.cs

@@ -0,0 +1,158 @@
+using OpenCvSharp;
+using Serilog;
+
+namespace Poe2Trade.Navigation;
+
+public class WorldMap : IDisposable
+{
+    private readonly MinimapConfig _config;
+    private readonly Mat _canvas;
+
+    public WorldMap(MinimapConfig config)
+    {
+        _config = config;
+        _canvas = new Mat(config.CanvasSize, config.CanvasSize, MatType.CV_8UC1, Scalar.Black);
+    }
+
+    public void StitchFrame(Mat classifiedMat, MapPosition position)
+    {
+        var halfSize = _config.CaptureSize / 2;
+        var canvasX = (int)Math.Round(position.X) - halfSize;
+        var canvasY = (int)Math.Round(position.Y) - halfSize;
+
+        // Clamp to canvas bounds
+        var srcX = Math.Max(0, -canvasX);
+        var srcY = Math.Max(0, -canvasY);
+        var dstX = Math.Max(0, canvasX);
+        var dstY = Math.Max(0, canvasY);
+        var w = Math.Min(_config.CaptureSize - srcX, _config.CanvasSize - dstX);
+        var h = Math.Min(_config.CaptureSize - srcY, _config.CanvasSize - dstY);
+
+        if (w <= 0 || h <= 0) return;
+
+        var srcRect = new Rect(srcX, srcY, w, h);
+        var dstRect = new Rect(dstX, dstY, w, h);
+
+        var srcRoi = new Mat(classifiedMat, srcRect);
+        var dstRoi = new Mat(_canvas, dstRect);
+
+        // Only paste non-Unknown pixels; don't overwrite Explored with Wall
+        for (var row = 0; row < h; row++)
+        {
+            for (var col = 0; col < w; col++)
+            {
+                var srcVal = srcRoi.At<byte>(row, col);
+                if (srcVal == (byte)MapCell.Unknown) continue;
+
+                var dstVal = dstRoi.At<byte>(row, col);
+                // Don't overwrite Explored with Wall (Explored is more reliable)
+                if (dstVal == (byte)MapCell.Explored && srcVal == (byte)MapCell.Wall)
+                    continue;
+
+                dstRoi.Set(row, col, srcVal);
+            }
+        }
+    }
+
+    public (double dirX, double dirY)? FindNearestUnexplored(MapPosition pos, int searchRadius = 200)
+    {
+        var cx = (int)Math.Round(pos.X);
+        var cy = (int)Math.Round(pos.Y);
+
+        // Scan in angular sectors to find direction with most Unknown cells at frontier
+        var bestAngle = double.NaN;
+        var bestScore = 0;
+        const int sectorCount = 16;
+        var fogRadius = _config.FogRadius;
+
+        for (var sector = 0; sector < sectorCount; sector++)
+        {
+            var angle = 2 * Math.PI * sector / sectorCount;
+            var score = 0;
+
+            // Sample along a cone in this direction, at the fog boundary and beyond
+            for (var r = fogRadius - 20; r <= fogRadius + searchRadius; r += 5)
+            {
+                for (var spread = -15; spread <= 15; spread += 5)
+                {
+                    var sampleAngle = angle + spread * Math.PI / 180;
+                    var sx = cx + (int)(r * Math.Cos(sampleAngle));
+                    var sy = cy + (int)(r * Math.Sin(sampleAngle));
+
+                    if (sx < 0 || sx >= _config.CanvasSize || sy < 0 || sy >= _config.CanvasSize)
+                        continue;
+
+                    if (_canvas.At<byte>(sy, sx) == (byte)MapCell.Unknown)
+                        score++;
+                }
+            }
+
+            if (score > bestScore)
+            {
+                bestScore = score;
+                bestAngle = angle;
+            }
+        }
+
+        if (bestScore == 0 || double.IsNaN(bestAngle))
+        {
+            Log.Information("No unexplored area found within search radius");
+            return null;
+        }
+
+        var dirX = Math.Cos(bestAngle);
+        var dirY = Math.Sin(bestAngle);
+        Log.Debug("Best exploration direction: angle={Angle:F1}deg score={Score}",
+            bestAngle * 180 / Math.PI, bestScore);
+        return (dirX, dirY);
+    }
+
+    public byte[] GetMapSnapshot()
+    {
+        Cv2.ImEncode(".png", _canvas, out var buf);
+        return buf;
+    }
+
+    public byte[] GetViewportSnapshot(MapPosition center, int viewSize = 400)
+    {
+        var cx = (int)Math.Round(center.X);
+        var cy = (int)Math.Round(center.Y);
+        var half = viewSize / 2;
+
+        // Clamp viewport to canvas
+        var x0 = Math.Clamp(cx - half, 0, _config.CanvasSize - viewSize);
+        var y0 = Math.Clamp(cy - half, 0, _config.CanvasSize - viewSize);
+        var roi = new Mat(_canvas, new Rect(x0, y0, viewSize, viewSize));
+
+        // Colorize: Unknown=#0d1117, Explored=#1f4068, Wall=#3d2d1a
+        using var colored = new Mat(viewSize, viewSize, MatType.CV_8UC3, new Scalar(23, 17, 13)); // BGR #0d1117
+        for (var r = 0; r < viewSize; r++)
+        for (var c = 0; c < viewSize; c++)
+        {
+            var v = roi.At<byte>(r, c);
+            if (v == (byte)MapCell.Explored)
+                colored.Set(r, c, new Vec3b(104, 64, 31)); // BGR #1f4068
+            else if (v == (byte)MapCell.Wall)
+                colored.Set(r, c, new Vec3b(26, 45, 61)); // BGR #3d2d1a
+        }
+
+        // Draw player dot
+        var px = cx - x0;
+        var py = cy - y0;
+        if (px >= 0 && px < viewSize && py >= 0 && py < viewSize)
+            Cv2.Circle(colored, new Point(px, py), 4, new Scalar(0, 140, 255), -1); // orange
+
+        Cv2.ImEncode(".png", colored, out var buf);
+        return buf;
+    }
+
+    public void Reset()
+    {
+        _canvas.SetTo(Scalar.Black);
+    }
+
+    public void Dispose()
+    {
+        _canvas.Dispose();
+    }
+}