446 lines
17 KiB
C#
446 lines
17 KiB
C#
using OpenCvSharp;
|
|
using Automata.Screen;
|
|
using Serilog;
|
|
using Region = Automata.Core.Region;
|
|
using Size = OpenCvSharp.Size;
|
|
|
|
namespace Automata.Navigation;
|
|
|
|
public class MinimapCapture : IFrameConsumer, IDisposable
|
|
{
|
|
private readonly MinimapConfig _config;
|
|
private readonly WallColorTracker _colorTracker;
|
|
private readonly IScreenCapture _backend; // kept for debug capture paths
|
|
private readonly IconDetector? _iconDetector;
|
|
private int _modeCheckCounter = 9; // trigger mode detection on first frame
|
|
private MinimapMode _detectedMode = MinimapMode.Overlay;
|
|
private int _pendingModeCount; // consecutive detections of a different mode
|
|
private MinimapMode _pendingMode;
|
|
private MinimapFrame? _lastFrame;
|
|
|
|
public MinimapMode DetectedMode => _detectedMode;
|
|
public MinimapFrame? LastFrame => _lastFrame;
|
|
|
|
/// <summary>
|
|
/// Atomically take ownership of the last frame (sets _lastFrame to null).
|
|
/// Caller is responsible for disposing the returned frame.
|
|
/// </summary>
|
|
public MinimapFrame? TakeFrame() => Interlocked.Exchange(ref _lastFrame, null);
|
|
|
|
public event Action<MinimapMode>? ModeChanged;
|
|
|
|
public MinimapCapture(MinimapConfig config, IScreenCapture backend, string? assetsDir = null)
|
|
{
|
|
_config = config;
|
|
_colorTracker = new WallColorTracker(config.WallLoHSV, config.WallHiHSV);
|
|
_backend = backend;
|
|
|
|
// Load icon templates if assets directory provided and contains templates
|
|
if (assetsDir != null && File.Exists(Path.Combine(assetsDir, "door.png")))
|
|
{
|
|
try { _iconDetector = new IconDetector(assetsDir); }
|
|
catch (Exception ex) { Log.Warning(ex, "Failed to load icon templates, icon detection disabled"); }
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// IFrameConsumer entry point — called by FramePipeline with the shared screen frame.
|
|
/// </summary>
|
|
public void Process(ScreenFrame screen)
|
|
{
|
|
// Auto-detect minimap mode by checking orange player dot at both positions.
|
|
// null = not in gameplay (loading/menu) → skip frame entirely.
|
|
// Require 3 consecutive detections of a new mode to avoid transient flips.
|
|
if (++_modeCheckCounter >= 10)
|
|
{
|
|
_modeCheckCounter = 0;
|
|
var detected = DetectMinimapMode(screen);
|
|
if (detected == null)
|
|
{
|
|
_pendingModeCount = 0;
|
|
_lastFrame = null;
|
|
return; // not in gameplay — skip frame
|
|
}
|
|
|
|
if (detected != _detectedMode)
|
|
{
|
|
if (detected == _pendingMode)
|
|
_pendingModeCount++;
|
|
else
|
|
{
|
|
_pendingMode = detected.Value;
|
|
_pendingModeCount = 1;
|
|
}
|
|
|
|
if (_pendingModeCount >= 3)
|
|
{
|
|
var oldMode = _detectedMode;
|
|
var oldRegion = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion;
|
|
_detectedMode = detected.Value;
|
|
var newRegion = _detectedMode == MinimapMode.Overlay ? _config.OverlayRegion : _config.CornerRegion;
|
|
Log.Information("MODE SWITCH: {Old} → {New} | oldRegion=({OX},{OY},{OW}x{OH}) newRegion=({NX},{NY},{NW}x{NH})",
|
|
oldMode, _detectedMode,
|
|
oldRegion.X, oldRegion.Y, oldRegion.Width, oldRegion.Height,
|
|
newRegion.X, newRegion.Y, newRegion.Width, newRegion.Height);
|
|
ResetAdaptation();
|
|
_pendingModeCount = 0;
|
|
ModeChanged?.Invoke(_detectedMode);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
_pendingModeCount = 0;
|
|
}
|
|
}
|
|
|
|
var region = _detectedMode == MinimapMode.Overlay
|
|
? _config.OverlayRegion
|
|
: _config.CornerRegion;
|
|
|
|
using var bgr = screen.CropBgr(region);
|
|
if (bgr.Empty())
|
|
return;
|
|
|
|
var frame = ProcessBgr(bgr);
|
|
if (frame == null) return;
|
|
|
|
Log.Debug("Process: mode={Mode} cropSize={W}x{H} classifiedSize={CW}x{CH} wallSize={WW}x{WH}",
|
|
_detectedMode, bgr.Width, bgr.Height,
|
|
frame.ClassifiedMat.Width, frame.ClassifiedMat.Height,
|
|
frame.WallMask.Width, frame.WallMask.Height);
|
|
|
|
var old = Interlocked.Exchange(ref _lastFrame, frame);
|
|
old?.Dispose();
|
|
}
|
|
|
|
private MinimapFrame? ProcessBgr(Mat bgr)
|
|
{
|
|
using var hsv = new Mat();
|
|
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
|
|
|
// Player mask (orange marker)
|
|
using var playerMask = new Mat();
|
|
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
|
var playerOffset = FindCentroid(playerMask);
|
|
|
|
// Wall mask: target #A2AEE5 blue-lavender structure lines (range adapts per-map)
|
|
var wallMask = BuildWallMask(hsv, playerMask, sample: true);
|
|
|
|
// Build classified mat (use actual frame size — differs between overlay and corner)
|
|
var frameSize = bgr.Width;
|
|
var classified = new Mat(frameSize, frameSize, MatType.CV_8UC1, Scalar.Black);
|
|
|
|
{
|
|
using var fogMask = BuildFogMask(hsv, wallMask, playerMask);
|
|
classified.SetTo(new Scalar((byte)MapCell.Fog), fogMask);
|
|
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
|
|
}
|
|
|
|
// Icon detection (overlay mode only — corner has different scale)
|
|
List<Point>? checkpointsOff = null;
|
|
List<Point>? checkpointsOn = null;
|
|
if (_detectedMode == MinimapMode.Overlay && _iconDetector != null)
|
|
{
|
|
using var grayForIcons = new Mat();
|
|
Cv2.CvtColor(bgr, grayForIcons, ColorConversionCodes.BGR2GRAY);
|
|
|
|
// Doors: stamp as Wall in classified + wallMask (fills gap so BFS blocks doors)
|
|
var doors = _iconDetector.DetectDoors(grayForIcons);
|
|
if (doors.Count > 0)
|
|
{
|
|
var doorSize = _iconDetector.DoorSize;
|
|
const int pad = 2;
|
|
foreach (var d in doors)
|
|
{
|
|
var rx = Math.Max(0, d.X - doorSize.Width / 2 - pad);
|
|
var ry = Math.Max(0, d.Y - doorSize.Height / 2 - pad);
|
|
var rw = Math.Min(doorSize.Width + 2 * pad, frameSize - rx);
|
|
var rh = Math.Min(doorSize.Height + 2 * pad, frameSize - ry);
|
|
if (rw <= 0 || rh <= 0) continue;
|
|
|
|
var rect = new Rect(rx, ry, rw, rh);
|
|
using var classRoi = new Mat(classified, rect);
|
|
classRoi.SetTo(new Scalar((byte)MapCell.Wall));
|
|
using var wallRoi = new Mat(wallMask, rect);
|
|
wallRoi.SetTo(new Scalar(255));
|
|
}
|
|
Log.Debug("Icons: {Count} doors stamped as wall", doors.Count);
|
|
}
|
|
|
|
// Checkpoints: don't stamp — just pass positions into MinimapFrame
|
|
checkpointsOff = _iconDetector.DetectCheckpointsOff(grayForIcons);
|
|
checkpointsOn = _iconDetector.DetectCheckpointsOn(grayForIcons);
|
|
if (checkpointsOff.Count > 0 || checkpointsOn.Count > 0)
|
|
Log.Debug("Icons: {Off} checkpoints-off, {On} checkpoints-on", checkpointsOff.Count, checkpointsOn.Count);
|
|
}
|
|
|
|
// Gray for correlation tracking (player zeroed)
|
|
var grayForCorr = new Mat();
|
|
Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY);
|
|
grayForCorr.SetTo(Scalar.Black, playerMask);
|
|
|
|
// Corner mode: rescale classified + wall mask to match overlay scale
|
|
if (_detectedMode == MinimapMode.Corner && Math.Abs(_config.CornerScale - 1.0) > 0.01)
|
|
{
|
|
var scaledSize = (int)Math.Round(frameSize * _config.CornerScale);
|
|
var scaledClassified = new Mat();
|
|
Cv2.Resize(classified, scaledClassified, new Size(scaledSize, scaledSize),
|
|
interpolation: InterpolationFlags.Nearest);
|
|
classified.Dispose();
|
|
classified = scaledClassified;
|
|
|
|
var scaledWalls = new Mat();
|
|
Cv2.Resize(wallMask, scaledWalls, new Size(scaledSize, scaledSize),
|
|
interpolation: InterpolationFlags.Nearest);
|
|
wallMask.Dispose();
|
|
wallMask = scaledWalls;
|
|
|
|
var scaledGray = new Mat();
|
|
Cv2.Resize(grayForCorr, scaledGray, new Size(scaledSize, scaledSize),
|
|
interpolation: InterpolationFlags.Linear);
|
|
grayForCorr.Dispose();
|
|
grayForCorr = scaledGray;
|
|
}
|
|
|
|
return new MinimapFrame(
|
|
GrayMat: grayForCorr,
|
|
WallMask: wallMask,
|
|
ClassifiedMat: classified,
|
|
PlayerOffset: playerOffset,
|
|
Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
|
CheckpointsOff: checkpointsOff,
|
|
CheckpointsOn: checkpointsOn
|
|
);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Detect minimap mode by sampling the orange player dot (#DE581B) at both
|
|
/// the overlay center and corner center. Returns null if neither is found
|
|
/// (loading screen, menu, map transition).
|
|
/// </summary>
|
|
private MinimapMode? DetectMinimapMode(ScreenFrame screen)
|
|
{
|
|
if (IsOrangeDot(screen, _config.OverlayCenterX, _config.OverlayCenterY))
|
|
return MinimapMode.Overlay;
|
|
if (IsOrangeDot(screen, _config.CornerCenterX, _config.CornerCenterY))
|
|
return MinimapMode.Corner;
|
|
return null;
|
|
}
|
|
|
|
private static bool IsOrangeDot(ScreenFrame screen, int cx, int cy)
|
|
{
|
|
if (cx < 2 || cy < 2 || cx + 2 >= screen.Width || cy + 2 >= screen.Height)
|
|
return false;
|
|
|
|
double bSum = 0, gSum = 0, rSum = 0;
|
|
var count = 0;
|
|
for (var dy = -2; dy <= 2; dy++)
|
|
for (var dx = -2; dx <= 2; dx++)
|
|
{
|
|
var px = screen.PixelAt(cx + dx, cy + dy);
|
|
bSum += px.Item0;
|
|
gSum += px.Item1;
|
|
rSum += px.Item2;
|
|
count++;
|
|
}
|
|
|
|
var r = rSum / count;
|
|
var g = gSum / count;
|
|
var b = bSum / count;
|
|
|
|
// #DE581B → R=222, G=88, B=27
|
|
const int tol = 60;
|
|
return Math.Abs(r - 222) < tol && Math.Abs(g - 88) < tol && Math.Abs(b - 27) < tol;
|
|
}
|
|
|
|
private Mat BuildWallMask(Mat hsv, Mat playerMask, bool sample = false)
|
|
{
|
|
var isCorner = _detectedMode == MinimapMode.Corner;
|
|
var lo = isCorner ? _config.CornerWallLoHSV : (_colorTracker.AdaptedLo ?? _config.WallLoHSV);
|
|
var hi = isCorner ? _config.CornerWallHiHSV : (_colorTracker.AdaptedHi ?? _config.WallHiHSV);
|
|
|
|
var wallMask = new Mat();
|
|
Cv2.InRange(hsv, lo, hi, wallMask);
|
|
|
|
using var notPlayer = new Mat();
|
|
Cv2.BitwiseNot(playerMask, notPlayer);
|
|
Cv2.BitwiseAnd(wallMask, notPlayer, wallMask);
|
|
|
|
// Overlay mode: exclude game text labels bleeding through the minimap.
|
|
// Text color #E0E0F6 → HSV(120, 23, 246): same blue hue as walls but
|
|
// much lower saturation. AA edges blend with the background pushing S
|
|
// above the wall floor. Detect the text core, dilate to cover the AA
|
|
// fringe, then subtract from wall mask before morphology amplifies it.
|
|
if (!isCorner)
|
|
{
|
|
using var textMask = new Mat();
|
|
Cv2.InRange(hsv, new Scalar(0, 0, 200), new Scalar(180, 50, 255), textMask);
|
|
using var textKernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(11, 11));
|
|
Cv2.Dilate(textMask, textMask, textKernel);
|
|
using var notText = new Mat();
|
|
Cv2.BitwiseNot(textMask, notText);
|
|
Cv2.BitwiseAnd(wallMask, notText, wallMask);
|
|
}
|
|
|
|
if (sample && !isCorner)
|
|
_colorTracker.SampleFrame(hsv, wallMask);
|
|
|
|
using var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3));
|
|
Cv2.MorphologyEx(wallMask, wallMask, MorphTypes.Close, kernel); // fill 1-2px gaps
|
|
Cv2.Dilate(wallMask, wallMask, kernel);
|
|
|
|
FilterSmallComponents(wallMask, _config.WallMinArea);
|
|
return wallMask;
|
|
}
|
|
|
|
private Mat BuildFogMask(Mat hsv, Mat wallMask, Mat playerMask)
|
|
{
|
|
using var allBlue = new Mat();
|
|
Cv2.InRange(hsv, _config.FogLoHSV, _config.FogHiHSV, allBlue);
|
|
|
|
using var notPlayer = new Mat();
|
|
Cv2.BitwiseNot(playerMask, notPlayer);
|
|
Cv2.BitwiseAnd(allBlue, notPlayer, allBlue);
|
|
|
|
var fogMask = new Mat();
|
|
using var notWalls = new Mat();
|
|
Cv2.BitwiseNot(wallMask, notWalls);
|
|
Cv2.BitwiseAnd(allBlue, notWalls, fogMask);
|
|
|
|
FilterSmallComponents(fogMask, _config.FogMinArea);
|
|
return fogMask;
|
|
}
|
|
|
|
private static void FilterSmallComponents(Mat mask, int minArea)
|
|
{
|
|
if (minArea <= 0) return;
|
|
|
|
using var labels = new Mat();
|
|
using var stats = new Mat();
|
|
using var centroids = new Mat();
|
|
var numLabels = Cv2.ConnectedComponentsWithStats(mask, labels, stats, centroids);
|
|
|
|
if (numLabels <= 1) return;
|
|
|
|
mask.SetTo(Scalar.Black);
|
|
for (var i = 1; i < numLabels; i++)
|
|
{
|
|
var area = stats.At<int>(i, 4);
|
|
if (area < minArea) continue;
|
|
|
|
using var comp = new Mat();
|
|
Cv2.Compare(labels, new Scalar(i), comp, CmpType.EQ);
|
|
mask.SetTo(new Scalar(255), comp);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Capture a single frame and return the requested pipeline stage as PNG bytes.
|
|
/// Uses CaptureRegion directly (debug path, not the pipeline).
|
|
/// </summary>
|
|
public byte[]? CaptureStage(MinimapDebugStage stage)
|
|
{
|
|
var region = _detectedMode == MinimapMode.Overlay
|
|
? _config.OverlayRegion
|
|
: _config.CornerRegion;
|
|
|
|
using var bgr = _backend.CaptureRegion(region);
|
|
if (bgr == null || bgr.Empty()) return null;
|
|
|
|
if (stage == MinimapDebugStage.Raw) return EncodePng(bgr);
|
|
|
|
using var hsv = new Mat();
|
|
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
|
|
|
if (stage is MinimapDebugStage.Hue or MinimapDebugStage.Saturation or MinimapDebugStage.Value)
|
|
{
|
|
var channels = Cv2.Split(hsv);
|
|
var idx = stage switch { MinimapDebugStage.Hue => 0, MinimapDebugStage.Saturation => 1, _ => 2 };
|
|
var result = EncodePng(channels[idx]);
|
|
foreach (var c in channels) c.Dispose();
|
|
return result;
|
|
}
|
|
|
|
using var playerMask = new Mat();
|
|
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
|
if (stage == MinimapDebugStage.Player) return EncodePng(playerMask);
|
|
|
|
using var wallMask = BuildWallMask(hsv, playerMask);
|
|
if (stage == MinimapDebugStage.Walls) return EncodePng(wallMask);
|
|
|
|
using var fogMask = BuildFogMask(hsv, wallMask, playerMask);
|
|
if (stage == MinimapDebugStage.Fog) return EncodePng(fogMask);
|
|
|
|
using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black);
|
|
classified.SetTo(new Scalar(180, 140, 70), fogMask);
|
|
classified.SetTo(new Scalar(26, 45, 61), wallMask);
|
|
classified.SetTo(new Scalar(0, 165, 255), playerMask);
|
|
return EncodePng(classified);
|
|
}
|
|
|
|
private static byte[] EncodePng(Mat mat)
|
|
{
|
|
Cv2.ImEncode(".png", mat, out var buf);
|
|
return buf;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Save debug images for tuning thresholds.
|
|
/// </summary>
|
|
public void SaveDebugCapture(string dir = "debug-minimap")
|
|
{
|
|
Directory.CreateDirectory(dir);
|
|
|
|
var region = _detectedMode == MinimapMode.Overlay
|
|
? _config.OverlayRegion
|
|
: _config.CornerRegion;
|
|
|
|
using var bgr = _backend.CaptureRegion(region);
|
|
if (bgr == null || bgr.Empty()) return;
|
|
|
|
using var hsv = new Mat();
|
|
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
|
|
|
using var playerMask = new Mat();
|
|
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
|
|
|
using var wallMask = BuildWallMask(hsv, playerMask);
|
|
|
|
using var classified = new Mat(bgr.Height, bgr.Width, MatType.CV_8UC3, Scalar.Black);
|
|
classified.SetTo(new Scalar(26, 45, 61), wallMask);
|
|
classified.SetTo(new Scalar(0, 165, 255), playerMask);
|
|
|
|
Cv2.ImWrite(Path.Combine(dir, "01-raw.png"), bgr);
|
|
Cv2.ImWrite(Path.Combine(dir, "02-walls.png"), wallMask);
|
|
Cv2.ImWrite(Path.Combine(dir, "03-player.png"), playerMask);
|
|
Cv2.ImWrite(Path.Combine(dir, "04-classified.png"), classified);
|
|
|
|
var channels = Cv2.Split(hsv);
|
|
Cv2.ImWrite(Path.Combine(dir, "05-hue.png"), channels[0]);
|
|
Cv2.ImWrite(Path.Combine(dir, "06-sat.png"), channels[1]);
|
|
Cv2.ImWrite(Path.Combine(dir, "07-val.png"), channels[2]);
|
|
foreach (var c in channels) c.Dispose();
|
|
|
|
Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir));
|
|
}
|
|
|
|
private Point2d FindCentroid(Mat mask)
|
|
{
|
|
var moments = Cv2.Moments(mask, true);
|
|
if (moments.M00 < 10)
|
|
return new Point2d(0, 0);
|
|
|
|
var cx = moments.M10 / moments.M00 - mask.Width / 2.0;
|
|
var cy = moments.M01 / moments.M00 - mask.Height / 2.0;
|
|
return new Point2d(cx, cy);
|
|
}
|
|
|
|
public void CommitWallColors() => _colorTracker.Commit();
|
|
public void ResetAdaptation() => _colorTracker.Reset();
|
|
|
|
public void Dispose()
|
|
{
|
|
_lastFrame?.Dispose();
|
|
_iconDetector?.Dispose();
|
|
}
|
|
}
|