minimap work

src/Poe2Trade.Navigation/MinimapCapture.cs

@@ -1,7 +1,6 @@
 using OpenCvSharp;
 using Serilog;
 using Region = Poe2Trade.Core.Region;
-using Point = OpenCvSharp.Point;
 using Size = OpenCvSharp.Size;
 
 namespace Poe2Trade.Navigation;
@@ -10,13 +9,12 @@ public class MinimapCapture : IDisposable
 {
     private readonly MinimapConfig _config;
     private readonly IScreenCapture _backend;
-    private Mat? _circularMask;
+    private readonly Queue<Mat> _frameBuffer = new();
 
     public MinimapCapture(MinimapConfig config)
     {
         _config = config;
         _backend = CreateBackend();
-        BuildCircularMask();
     }
 
     private static IScreenCapture CreateBackend()
@@ -48,14 +46,6 @@ public class MinimapCapture : IDisposable
         return new GdiCapture();
     }
 
-    private void BuildCircularMask()
-    {
-        var size = _config.CaptureSize;
-        _circularMask = new Mat(size, size, MatType.CV_8UC1, Scalar.Black);
-        var center = new Point(size / 2, size / 2);
-        Cv2.Circle(_circularMask, center, _config.FogRadius, Scalar.White, -1);
-    }
-
     public MinimapFrame? CaptureFrame()
     {
         var region = _config.CaptureRegion;
@@ -64,63 +54,207 @@ public class MinimapCapture : IDisposable
         if (bgr == null || bgr.Empty())
             return null;
 
-        // Apply circular mask to ignore area outside fog-of-war circle
-        using var masked = new Mat();
-        Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
-
-        // Convert to HSV
+        // --- 1. HSV + extract S/V channels ---
         using var hsv = new Mat();
-        Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
-
-        // Classify explored areas
-        using var exploredMask = new Mat();
-        Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
-
-        // Classify walls: dark pixels within the circular mask
+        Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
+        using var satChan = new Mat();
         using var valueChan = new Mat();
-        Cv2.ExtractChannel(hsv, valueChan, 2); // V channel
-        using var darkMask = new Mat();
-        Cv2.Threshold(valueChan, darkMask, _config.WallMaxValue, 255, ThresholdTypes.BinaryInv);
-        // Apply morphological close to connect wall fragments
-        using var wallKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
-        using var wallMask = new Mat();
-        Cv2.MorphologyEx(darkMask, wallMask, MorphTypes.Close, wallKernel);
-        // Only within circular mask
-        Cv2.BitwiseAnd(wallMask, _circularMask!, wallMask);
-        // Don't count explored pixels as wall
-        using var notExplored = new Mat();
-        Cv2.BitwiseNot(exploredMask, notExplored);
-        Cv2.BitwiseAnd(wallMask, notExplored, wallMask);
+        Cv2.ExtractChannel(hsv, satChan, 1);   // S
+        Cv2.ExtractChannel(hsv, valueChan, 2); // V
 
-        // Build classified mat: Unknown=0, Explored=1, Wall=2
-        var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
-        classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
-        classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
-        // Ensure only within circular mask
-        Cv2.BitwiseAnd(classified, _circularMask!, classified);
-
-        // Detect player marker (orange X)
+        // --- 2. Player mask (orange marker) ---
         using var playerMask = new Mat();
         Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
         var playerOffset = FindCentroid(playerMask);
 
-        // Convert to grayscale for phase correlation
-        var gray = new Mat();
-        Cv2.CvtColor(masked, gray, ColorConversionCodes.BGR2GRAY);
-        // Apply circular mask to gray too
-        Cv2.BitwiseAnd(gray, _circularMask!, gray);
+        // --- 3. Wall mask: bright OR saturated → structure lines ---
+        using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
+
+        // --- 4. Explored mask: brightness above fog-of-war, minus walls/player ---
+        using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
+
+        // --- 5. Build raw classified mat ---
+        var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
+        classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
+        classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
+
+        // --- 6. Temporal smoothing: majority vote ---
+        var smoothed = TemporalSmooth(classified); // classified goes into ring buffer
+
+        // --- 7. Gray for phase correlation (walls zeroed out — structural noise) ---
+        var grayForCorr = new Mat();
+        Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY);
+        grayForCorr.SetTo(Scalar.Black, wallMask);
 
         return new MinimapFrame(
-            GrayMat: gray,
-            ClassifiedMat: classified,
+            GrayMat: grayForCorr,
+            ClassifiedMat: smoothed,
             PlayerOffset: playerOffset,
             Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
         );
     }
 
+    private Mat BuildWallMask(Mat satChan, Mat valueChan, Mat playerMask)
+    {
+        // Wall = bright OR saturated pixels (minimap structure lines, icons)
+        using var highV = new Mat();
+        Cv2.Threshold(valueChan, highV, _config.WallMinValue, 255, ThresholdTypes.Binary);
+        using var highS = new Mat();
+        Cv2.Threshold(satChan, highS, _config.WallMinSat, 255, ThresholdTypes.Binary);
+
+        var wallMask = new Mat();
+        Cv2.BitwiseOr(highV, highS, wallMask);
+
+        // Dilate to connect wall line fragments
+        using var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3));
+        Cv2.Dilate(wallMask, wallMask, kernel);
+
+        // Subtract player (bright/saturated marker would otherwise be classified as wall)
+        using var notPlayer = new Mat();
+        Cv2.BitwiseNot(playerMask, notPlayer);
+        Cv2.BitwiseAnd(wallMask, notPlayer, wallMask);
+
+        FilterSmallComponents(wallMask, _config.WallMinArea);
+        return wallMask;
+    }
+
+    private Mat BuildExploredMask(Mat valueChan, Mat wallMask, Mat playerMask)
+    {
+        // Explored = any pixel above fog-of-war darkness, minus walls and player
+        var exploredMask = new Mat();
+        Cv2.Threshold(valueChan, exploredMask, _config.FloorMinValue, 255, ThresholdTypes.Binary);
+
+        using var notWall = new Mat();
+        Cv2.BitwiseNot(wallMask, notWall);
+        Cv2.BitwiseAnd(exploredMask, notWall, exploredMask);
+
+        using var notPlayer = new Mat();
+        Cv2.BitwiseNot(playerMask, notPlayer);
+        Cv2.BitwiseAnd(exploredMask, notPlayer, exploredMask);
+
+        return exploredMask;
+    }
+
+    private static void FilterSmallComponents(Mat mask, int minArea)
+    {
+        if (minArea <= 0) return;
+
+        using var labels = new Mat();
+        using var stats = new Mat();
+        using var centroids = new Mat();
+        var numLabels = Cv2.ConnectedComponentsWithStats(mask, labels, stats, centroids);
+
+        if (numLabels <= 1) return; // only background
+
+        // Clear mask, then re-add only components meeting min area
+        mask.SetTo(Scalar.Black);
+        for (var i = 1; i < numLabels; i++)
+        {
+            var area = stats.At<int>(i, 4); // CC_STAT_AREA
+            if (area < minArea) continue;
+
+            using var comp = new Mat();
+            Cv2.Compare(labels, new Scalar(i), comp, CmpType.EQ);
+            mask.SetTo(new Scalar(255), comp);
+        }
+    }
+
+    private Mat TemporalSmooth(Mat classified)
+    {
+        // Add raw frame to ring buffer (takes ownership)
+        _frameBuffer.Enqueue(classified);
+        while (_frameBuffer.Count > _config.TemporalFrameCount)
+            _frameBuffer.Dequeue().Dispose();
+
+        // Not enough frames yet — return as-is
+        if (_frameBuffer.Count < 2)
+            return classified.Clone();
+
+        var size = classified.Size();
+        using var wallCount = new Mat(size, MatType.CV_8UC1, Scalar.Black);
+        using var exploredCount = new Mat(size, MatType.CV_8UC1, Scalar.Black);
+
+        foreach (var frame in _frameBuffer)
+        {
+            using var isWall = new Mat();
+            Cv2.Compare(frame, new Scalar((byte)MapCell.Wall), isWall, CmpType.EQ);
+            Cv2.Add(wallCount, new Scalar(1), wallCount, isWall);
+
+            using var isExplored = new Mat();
+            Cv2.Compare(frame, new Scalar((byte)MapCell.Explored), isExplored, CmpType.EQ);
+            Cv2.Add(exploredCount, new Scalar(1), exploredCount, isExplored);
+        }
+
+        // Apply vote thresholds
+        using var wallPass = new Mat();
+        Cv2.Threshold(wallCount, wallPass,
+            _config.WallTemporalThreshold - 0.5, 255, ThresholdTypes.Binary);
+        using var exploredPass = new Mat();
+        Cv2.Threshold(exploredCount, exploredPass,
+            _config.ExploredTemporalThreshold - 0.5, 255, ThresholdTypes.Binary);
+
+        // Build smoothed result (explored wins over wall)
+        var result = new Mat(size, MatType.CV_8UC1, Scalar.Black);
+        result.SetTo(new Scalar((byte)MapCell.Wall), wallPass);
+        result.SetTo(new Scalar((byte)MapCell.Explored), exploredPass);
+
+        return result;
+    }
+
     /// <summary>
-    /// Save debug images: raw capture, HSV mask, classified result.
-    /// Call once to diagnose color ranges.
+    /// Capture a single frame and return the requested pipeline stage as PNG bytes.
+    /// </summary>
+    public byte[]? CaptureStage(MinimapDebugStage stage)
+    {
+        var region = _config.CaptureRegion;
+        using var bgr = _backend.CaptureRegion(region);
+        if (bgr == null || bgr.Empty()) return null;
+
+        if (stage == MinimapDebugStage.Raw) return EncodePng(bgr);
+
+        using var hsv = new Mat();
+        Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
+
+        if (stage is MinimapDebugStage.Hue or MinimapDebugStage.Saturation or MinimapDebugStage.Value)
+        {
+            var channels = Cv2.Split(hsv);
+            var idx = stage switch { MinimapDebugStage.Hue => 0, MinimapDebugStage.Saturation => 1, _ => 2 };
+            var result = EncodePng(channels[idx]);
+            foreach (var c in channels) c.Dispose();
+            return result;
+        }
+
+        using var satChan = new Mat();
+        using var valueChan = new Mat();
+        Cv2.ExtractChannel(hsv, satChan, 1);
+        Cv2.ExtractChannel(hsv, valueChan, 2);
+
+        using var playerMask = new Mat();
+        Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
+        if (stage == MinimapDebugStage.Player) return EncodePng(playerMask);
+
+        using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
+        if (stage == MinimapDebugStage.Walls) return EncodePng(wallMask);
+
+        using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
+        if (stage == MinimapDebugStage.Explored) return EncodePng(exploredMask);
+
+        // Classified
+        using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
+        classified.SetTo(new Scalar(104, 64, 31), exploredMask);
+        classified.SetTo(new Scalar(26, 45, 61), wallMask);
+        classified.SetTo(new Scalar(0, 165, 255), playerMask);
+        return EncodePng(classified);
+    }
+
+    private static byte[] EncodePng(Mat mat)
+    {
+        Cv2.ImEncode(".png", mat, out var buf);
+        return buf;
+    }
+
+    /// <summary>
+    /// Save debug images for tuning thresholds.
     /// </summary>
     public void SaveDebugCapture(string dir = "debug-minimap")
     {
@@ -129,29 +263,37 @@ public class MinimapCapture : IDisposable
         using var bgr = _backend.CaptureRegion(region);
         if (bgr == null || bgr.Empty()) return;
 
-        using var masked = new Mat();
-        Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
-
         using var hsv = new Mat();
-        Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
+        Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
 
-        using var exploredMask = new Mat();
-        Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
+        using var satChan = new Mat();
+        using var valueChan = new Mat();
+        Cv2.ExtractChannel(hsv, satChan, 1);
+        Cv2.ExtractChannel(hsv, valueChan, 2);
 
         using var playerMask = new Mat();
         Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
 
-        // Save raw, masked, explored filter, player filter
-        Cv2.ImWrite(Path.Combine(dir, "1-raw.png"), bgr);
-        Cv2.ImWrite(Path.Combine(dir, "2-masked.png"), masked);
-        Cv2.ImWrite(Path.Combine(dir, "3-explored.png"), exploredMask);
-        Cv2.ImWrite(Path.Combine(dir, "4-player.png"), playerMask);
+        using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
+        using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
 
-        // Save HSV channels separately for tuning
+        // Colorized classified
+        using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
+        classified.SetTo(new Scalar(104, 64, 31), exploredMask);   // blue
+        classified.SetTo(new Scalar(26, 45, 61), wallMask);         // brown
+        classified.SetTo(new Scalar(0, 165, 255), playerMask);      // orange
+
+        Cv2.ImWrite(Path.Combine(dir, "01-raw.png"), bgr);
+        Cv2.ImWrite(Path.Combine(dir, "02-walls.png"), wallMask);
+        Cv2.ImWrite(Path.Combine(dir, "03-explored.png"), exploredMask);
+        Cv2.ImWrite(Path.Combine(dir, "04-player.png"), playerMask);
+        Cv2.ImWrite(Path.Combine(dir, "05-classified.png"), classified);
+
+        // HSV channels
         var channels = Cv2.Split(hsv);
-        Cv2.ImWrite(Path.Combine(dir, "5-hue.png"), channels[0]);
-        Cv2.ImWrite(Path.Combine(dir, "6-sat.png"), channels[1]);
-        Cv2.ImWrite(Path.Combine(dir, "7-val.png"), channels[2]);
+        Cv2.ImWrite(Path.Combine(dir, "06-hue.png"), channels[0]);
+        Cv2.ImWrite(Path.Combine(dir, "07-sat.png"), channels[1]);
+        Cv2.ImWrite(Path.Combine(dir, "08-val.png"), channels[2]);
         foreach (var c in channels) c.Dispose();
 
         Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir));
@@ -170,7 +312,8 @@ public class MinimapCapture : IDisposable
 
     public void Dispose()
     {
-        _circularMask?.Dispose();
         _backend.Dispose();
+        while (_frameBuffer.Count > 0)
+            _frameBuffer.Dequeue().Dispose();
     }
 }