using OpenCvSharp; using Automata.Screen; using Serilog; using Region = Automata.Core.Region; using Size = OpenCvSharp.Size; namespace Automata.Navigation; public class MinimapCapture : IFrameConsumer, IDisposable { private readonly MinimapConfig _config; private readonly WallColorTracker _colorTracker; private readonly IScreenCapture _backend; // kept for debug capture paths private readonly IconDetector? _iconDetector; private int _modeCheckCounter = 9; // trigger mode detection on first frame private MinimapMode _detectedMode = MinimapMode.Overlay; private int _pendingModeCount; // consecutive detections of a different mode private MinimapMode _pendingMode; private MinimapFrame? _lastFrame; public MinimapMode DetectedMode => _detectedMode; public MinimapFrame? LastFrame => _lastFrame; /// /// Atomically take ownership of the last frame (sets _lastFrame to null). /// Caller is responsible for disposing the returned frame. /// public MinimapFrame? TakeFrame() => Interlocked.Exchange(ref _lastFrame, null); public event Action? ModeChanged; public MinimapCapture(MinimapConfig config, IScreenCapture backend, string? assetsDir = null) { _config = config; _colorTracker = new WallColorTracker(config.WallLoHSV, config.WallHiHSV); _backend = backend; // Load icon templates if assets directory provided and contains templates if (assetsDir != null && File.Exists(Path.Combine(assetsDir, "door.png"))) { try { _iconDetector = new IconDetector(assetsDir); } catch (Exception ex) { Log.Warning(ex, "Failed to load icon templates, icon detection disabled"); } } } /// /// IFrameConsumer entry point — called by FramePipeline with the shared screen frame. /// public void Process(ScreenFrame screen) { // Auto-detect minimap mode by checking orange player dot at both positions. // null = not in gameplay (loading/menu) → skip frame entirely. // Require 3 consecutive detections of a new mode to avoid transient flips. if (++_modeCheckCounter >= 10) { _modeCheckCounter = 0; var detected = DetectMinimapMode(screen); if (detected == null) { _pendingModeCount = 0; _lastFrame = null; return; // not in gameplay — skip frame } if (detected != _detectedMode) { if (detected == _pendingMode) _pendingModeCount++; else { _pendingMode = detected.Value; _pendingModeCount = 1; } if (_pendingModeCount >= 3) { var oldMode = _detectedMode; var oldRegion = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion; _detectedMode = detected.Value; var newRegion = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion; Log.Information("MODE SWITCH: {Old} → {New} | oldRegion=({OX},{OY},{OW}x{OH}) newRegion=({NX},{NY},{NW}x{NH})", oldMode, _detectedMode, oldRegion.X, oldRegion.Y, oldRegion.Width, oldRegion.Height, newRegion.X, newRegion.Y, newRegion.Width, newRegion.Height); ResetAdaptation(); _pendingModeCount = 0; ModeChanged?.Invoke(_detectedMode); } } else { _pendingModeCount = 0; } } var region = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion; using var bgr = screen.CropBgr(region); if (bgr.Empty()) return; var frame = ProcessBgr(bgr); if (frame == null) return; Log.Debug("Process: mode={Mode} cropSize={W}x{H} classifiedSize={CW}x{CH} wallSize={WW}x{WH}", _detectedMode, bgr.Width, bgr.Height, frame.ClassifiedMat.Width, frame.ClassifiedMat.Height, frame.WallMask.Width, frame.WallMask.Height); var old = Interlocked.Exchange(ref _lastFrame, frame); old?.Dispose(); } private MinimapFrame? ProcessBgr(Mat bgr) { using var hsv = new Mat(); Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV); // Player mask (orange marker) using var playerMask = new Mat(); Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask); var playerOffset = FindCentroid(playerMask); // Wall mask: target #A2AEE5 blue-lavender structure lines (range adapts per-map) var wallMask = BuildWallMask(hsv, playerMask, sample: true); // Build classified mat (use actual frame size — differs between overlay and corner) var frameSize = bgr.Width; var classified = new Mat(frameSize, frameSize, MatType.CV_8UC1, Scalar.Black); { using var fogMask = BuildFogMask(hsv, wallMask, playerMask); classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask); classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask); } // Icon detection (overlay mode only — corner has different scale) List? checkpointsOff = null; List? checkpointsOn = null; if (_detectedMode == MinimapMode.Overlay && _iconDetector != null) { using var grayForIcons = new Mat(); Cv2.CvtColor(bgr, grayForIcons, ColorConversionCodes.BGR2GRAY); // Doors: stamp as Wall in classified + wallMask (fills gap so BFS blocks doors) var doors = _iconDetector.DetectDoors(grayForIcons); if (doors.Count > 0) { var doorSize = _iconDetector.DoorSize; const int pad = 2; foreach (var d in doors) { var rx = Math.Max(0, d.X - doorSize.Width / 2 - pad); var ry = Math.Max(0, d.Y - doorSize.Height / 2 - pad); var rw = Math.Min(doorSize.Width + 2 * pad, frameSize - rx); var rh = Math.Min(doorSize.Height + 2 * pad, frameSize - ry); if (rw <= 0 || rh <= 0) continue; var rect = new Rect(rx, ry, rw, rh); using var classRoi = new Mat(classified, rect); classRoi.SetTo(new Scalar((byte)MapCell.Wall)); using var wallRoi = new Mat(wallMask, rect); wallRoi.SetTo(new Scalar(255)); } Log.Debug("Icons: {Count} doors stamped as wall", doors.Count); } // Checkpoints: don't stamp — just pass positions into MinimapFrame checkpointsOff = _iconDetector.DetectCheckpointsOff(grayForIcons); checkpointsOn = _iconDetector.DetectCheckpointsOn(grayForIcons); if (checkpointsOff.Count > 0 || checkpointsOn.Count > 0) Log.Debug("Icons: {Off} checkpoints-off, {On} checkpoints-on", checkpointsOff.Count, checkpointsOn.Count); } // Gray for correlation tracking (player zeroed) var grayForCorr = new Mat(); Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY); grayForCorr.SetTo(Scalar.Black, playerMask); // Corner mode: rescale classified + wall mask to match overlay scale if (_detectedMode == MinimapMode.Corner && Math.Abs(_config.CornerScale - 1.0) > 0.01) { var scaledSize = (int)Math.Round(frameSize * _config.CornerScale); var scaledClassified = new Mat(); Cv2.Resize(classified, scaledClassified, new Size(scaledSize, scaledSize), interpolation: InterpolationFlags.Nearest); classified.Dispose(); classified = scaledClassified; var scaledWalls = new Mat(); Cv2.Resize(wallMask, scaledWalls, new Size(scaledSize, scaledSize), interpolation: InterpolationFlags.Nearest); wallMask.Dispose(); wallMask = scaledWalls; var scaledGray = new Mat(); Cv2.Resize(grayForCorr, scaledGray, new Size(scaledSize, scaledSize), interpolation: InterpolationFlags.Linear); grayForCorr.Dispose(); grayForCorr = scaledGray; } return new MinimapFrame( GrayMat: grayForCorr, WallMask: wallMask, ClassifiedMat: classified, PlayerOffset: playerOffset, Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), CheckpointsOff: checkpointsOff, CheckpointsOn: checkpointsOn ); } /// /// Detect minimap mode by sampling the orange player dot (#DE581B) at both /// the overlay center and corner center. Returns null if neither is found /// (loading screen, menu, map transition). /// private MinimapMode? DetectMinimapMode(ScreenFrame screen) { if (IsOrangeDot(screen, _config.OverlayCenterX, _config.OverlayCenterY)) return MinimapMode.Overlay; if (IsOrangeDot(screen, _config.CornerCenterX, _config.CornerCenterY)) return MinimapMode.Corner; return null; } private static bool IsOrangeDot(ScreenFrame screen, int cx, int cy) { if (cx < 2 || cy < 2 || cx + 2 >= screen.Width || cy + 2 >= screen.Height) return false; double bSum = 0, gSum = 0, rSum = 0; var count = 0; for (var dy = -2; dy <= 2; dy++) for (var dx = -2; dx <= 2; dx++) { var px = screen.PixelAt(cx + dx, cy + dy); bSum += px.Item0; gSum += px.Item1; rSum += px.Item2; count++; } var r = rSum / count; var g = gSum / count; var b = bSum / count; // #DE581B → R=222, G=88, B=27 const int tol = 60; return Math.Abs(r - 222) < tol && Math.Abs(g - 88) < tol && Math.Abs(b - 27) < tol; } private Mat BuildWallMask(Mat hsv, Mat playerMask, bool sample = false) { var isCorner = _detectedMode == MinimapMode.Corner; var lo = isCorner ? _config.CornerWallLoHSV : (_colorTracker.AdaptedLo ?? _config.WallLoHSV); var hi = isCorner ? _config.CornerWallHiHSV : (_colorTracker.AdaptedHi ?? _config.WallHiHSV); var wallMask = new Mat(); Cv2.InRange(hsv, lo, hi, wallMask); using var notPlayer = new Mat(); Cv2.BitwiseNot(playerMask, notPlayer); Cv2.BitwiseAnd(wallMask, notPlayer, wallMask); // Overlay mode: exclude game text labels bleeding through the minimap. // Text color #E0E0F6 → HSV(120, 23, 246): same blue hue as walls but // much lower saturation. AA edges blend with the background pushing S // above the wall floor. Detect the text core, dilate to cover the AA // fringe, then subtract from wall mask before morphology amplifies it. if (!isCorner) { using var textMask = new Mat(); Cv2.InRange(hsv, new Scalar(0, 0, 200), new Scalar(180, 50, 255), textMask); using var textKernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(11, 11)); Cv2.Dilate(textMask, textMask, textKernel); using var notText = new Mat(); Cv2.BitwiseNot(textMask, notText); Cv2.BitwiseAnd(wallMask, notText, wallMask); } if (sample && !isCorner) _colorTracker.SampleFrame(hsv, wallMask); using var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3)); Cv2.MorphologyEx(wallMask, wallMask, MorphTypes.Close, kernel); // fill 1-2px gaps Cv2.Dilate(wallMask, wallMask, kernel); FilterSmallComponents(wallMask, _config.WallMinArea); return wallMask; } private Mat BuildFogMask(Mat hsv, Mat wallMask, Mat playerMask) { using var allBlue = new Mat(); Cv2.InRange(hsv, _config.FogLoHSV, _config.FogHiHSV, allBlue); using var notPlayer = new Mat(); Cv2.BitwiseNot(playerMask, notPlayer); Cv2.BitwiseAnd(allBlue, notPlayer, allBlue); var fogMask = new Mat(); using var notWalls = new Mat(); Cv2.BitwiseNot(wallMask, notWalls); Cv2.BitwiseAnd(allBlue, notWalls, fogMask); FilterSmallComponents(fogMask, _config.FogMinArea); return fogMask; } private static void FilterSmallComponents(Mat mask, int minArea) { if (minArea <= 0) return; using var labels = new Mat(); using var stats = new Mat(); using var centroids = new Mat(); var numLabels = Cv2.ConnectedComponentsWithStats(mask, labels, stats, centroids); if (numLabels <= 1) return; mask.SetTo(Scalar.Black); for (var i = 1; i < numLabels; i++) { var area = stats.At(i, 4); if (area < minArea) continue; using var comp = new Mat(); Cv2.Compare(labels, new Scalar(i), comp, CmpType.EQ); mask.SetTo(new Scalar(255), comp); } } /// /// Capture a single frame and return the requested pipeline stage as PNG bytes. /// Uses CaptureRegion directly (debug path, not the pipeline). /// public byte[]? CaptureStage(MinimapDebugStage stage) { var region = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion; using var bgr = _backend.CaptureRegion(region); if (bgr == null || bgr.Empty()) return null; if (stage == MinimapDebugStage.Raw) return EncodePng(bgr); using var hsv = new Mat(); Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV); if (stage is MinimapDebugStage.Hue or MinimapDebugStage.Saturation or MinimapDebugStage.Value) { var channels = Cv2.Split(hsv); var idx = stage switch { MinimapDebugStage.Hue => 0, MinimapDebugStage.Saturation => 1, _ => 2 }; var result = EncodePng(channels[idx]); foreach (var c in channels) c.Dispose(); return result; } using var playerMask = new Mat(); Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask); if (stage == MinimapDebugStage.Player) return EncodePng(playerMask); using var wallMask = BuildWallMask(hsv, playerMask); if (stage == MinimapDebugStage.Walls) return EncodePng(wallMask); using var fogMask = BuildFogMask(hsv, wallMask, playerMask); if (stage == MinimapDebugStage.Fog) return EncodePng(fogMask); using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black); classified.SetTo(new Scalar(180, 140, 70), fogMask); classified.SetTo(new Scalar(26, 45, 61), wallMask); classified.SetTo(new Scalar(0, 165, 255), playerMask); return EncodePng(classified); } private static byte[] EncodePng(Mat mat) { Cv2.ImEncode(".png", mat, out var buf); return buf; } /// /// Save debug images for tuning thresholds. /// public void SaveDebugCapture(string dir = "debug-minimap") { Directory.CreateDirectory(dir); var region = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion; using var bgr = _backend.CaptureRegion(region); if (bgr == null || bgr.Empty()) return; using var hsv = new Mat(); Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV); using var playerMask = new Mat(); Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask); using var wallMask = BuildWallMask(hsv, playerMask); using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black); classified.SetTo(new Scalar(26, 45, 61), wallMask); classified.SetTo(new Scalar(0, 165, 255), playerMask); Cv2.ImWrite(Path.Combine(dir, "01-raw.png"), bgr); Cv2.ImWrite(Path.Combine(dir, "02-walls.png"), wallMask); Cv2.ImWrite(Path.Combine(dir, "03-player.png"), playerMask); Cv2.ImWrite(Path.Combine(dir, "04-classified.png"), classified); var channels = Cv2.Split(hsv); Cv2.ImWrite(Path.Combine(dir, "05-hue.png"), channels[0]); Cv2.ImWrite(Path.Combine(dir, "06-sat.png"), channels[1]); Cv2.ImWrite(Path.Combine(dir, "07-val.png"), channels[2]); foreach (var c in channels) c.Dispose(); Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir)); } private Point2d FindCentroid(Mat mask) { var moments = Cv2.Moments(mask, true); if (moments.M00 < 10) return new Point2d(0, 0); var cx = moments.M10 / moments.M00 - mask.Width / 2.0; var cy = moments.M01 / moments.M00 - mask.Height / 2.0; return new Point2d(cx, cy); } public void CommitWallColors() => _colorTracker.Commit(); public void ResetAdaptation() => _colorTracker.Reset(); public void Dispose() { _lastFrame?.Dispose(); _iconDetector?.Dispose(); } }