minimap work

This commit is contained in:
Boki 2026-02-13 12:14:57 -05:00
parent 802f1030d5
commit ba8626b470
7 changed files with 312 additions and 118 deletions

View file

@ -1,7 +1,6 @@
using OpenCvSharp;
using Serilog;
using Region = Poe2Trade.Core.Region;
using Point = OpenCvSharp.Point;
using Size = OpenCvSharp.Size;
namespace Poe2Trade.Navigation;
@ -10,13 +9,12 @@ public class MinimapCapture : IDisposable
{
private readonly MinimapConfig _config;
private readonly IScreenCapture _backend;
private Mat? _circularMask;
private readonly Queue<Mat> _frameBuffer = new();
public MinimapCapture(MinimapConfig config)
{
_config = config;
_backend = CreateBackend();
BuildCircularMask();
}
private static IScreenCapture CreateBackend()
@ -48,14 +46,6 @@ public class MinimapCapture : IDisposable
return new GdiCapture();
}
private void BuildCircularMask()
{
var size = _config.CaptureSize;
_circularMask = new Mat(size, size, MatType.CV_8UC1, Scalar.Black);
var center = new Point(size / 2, size / 2);
Cv2.Circle(_circularMask, center, _config.FogRadius, Scalar.White, -1);
}
public MinimapFrame? CaptureFrame()
{
var region = _config.CaptureRegion;
@ -64,63 +54,207 @@ public class MinimapCapture : IDisposable
if (bgr == null || bgr.Empty())
return null;
// Apply circular mask to ignore area outside fog-of-war circle
using var masked = new Mat();
Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
// Convert to HSV
// --- 1. HSV + extract S/V channels ---
using var hsv = new Mat();
Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
// Classify explored areas
using var exploredMask = new Mat();
Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
// Classify walls: dark pixels within the circular mask
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
using var satChan = new Mat();
using var valueChan = new Mat();
Cv2.ExtractChannel(hsv, valueChan, 2); // V channel
using var darkMask = new Mat();
Cv2.Threshold(valueChan, darkMask, _config.WallMaxValue, 255, ThresholdTypes.BinaryInv);
// Apply morphological close to connect wall fragments
using var wallKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
using var wallMask = new Mat();
Cv2.MorphologyEx(darkMask, wallMask, MorphTypes.Close, wallKernel);
// Only within circular mask
Cv2.BitwiseAnd(wallMask, _circularMask!, wallMask);
// Don't count explored pixels as wall
using var notExplored = new Mat();
Cv2.BitwiseNot(exploredMask, notExplored);
Cv2.BitwiseAnd(wallMask, notExplored, wallMask);
Cv2.ExtractChannel(hsv, satChan, 1); // S
Cv2.ExtractChannel(hsv, valueChan, 2); // V
// Build classified mat: Unknown=0, Explored=1, Wall=2
var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
// Ensure only within circular mask
Cv2.BitwiseAnd(classified, _circularMask!, classified);
// Detect player marker (orange X)
// --- 2. Player mask (orange marker) ---
using var playerMask = new Mat();
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
var playerOffset = FindCentroid(playerMask);
// Convert to grayscale for phase correlation
var gray = new Mat();
Cv2.CvtColor(masked, gray, ColorConversionCodes.BGR2GRAY);
// Apply circular mask to gray too
Cv2.BitwiseAnd(gray, _circularMask!, gray);
// --- 3. Wall mask: bright OR saturated → structure lines ---
using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
// --- 4. Explored mask: brightness above fog-of-war, minus walls/player ---
using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
// --- 5. Build raw classified mat ---
var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
// --- 6. Temporal smoothing: majority vote ---
var smoothed = TemporalSmooth(classified); // classified goes into ring buffer
// --- 7. Gray for phase correlation (walls zeroed out — structural noise) ---
var grayForCorr = new Mat();
Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY);
grayForCorr.SetTo(Scalar.Black, wallMask);
return new MinimapFrame(
GrayMat: gray,
ClassifiedMat: classified,
GrayMat: grayForCorr,
ClassifiedMat: smoothed,
PlayerOffset: playerOffset,
Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
);
}
private Mat BuildWallMask(Mat satChan, Mat valueChan, Mat playerMask)
{
// Wall = bright OR saturated pixels (minimap structure lines, icons)
using var highV = new Mat();
Cv2.Threshold(valueChan, highV, _config.WallMinValue, 255, ThresholdTypes.Binary);
using var highS = new Mat();
Cv2.Threshold(satChan, highS, _config.WallMinSat, 255, ThresholdTypes.Binary);
var wallMask = new Mat();
Cv2.BitwiseOr(highV, highS, wallMask);
// Dilate to connect wall line fragments
using var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3));
Cv2.Dilate(wallMask, wallMask, kernel);
// Subtract player (bright/saturated marker would otherwise be classified as wall)
using var notPlayer = new Mat();
Cv2.BitwiseNot(playerMask, notPlayer);
Cv2.BitwiseAnd(wallMask, notPlayer, wallMask);
FilterSmallComponents(wallMask, _config.WallMinArea);
return wallMask;
}
private Mat BuildExploredMask(Mat valueChan, Mat wallMask, Mat playerMask)
{
// Explored = any pixel above fog-of-war darkness, minus walls and player
var exploredMask = new Mat();
Cv2.Threshold(valueChan, exploredMask, _config.FloorMinValue, 255, ThresholdTypes.Binary);
using var notWall = new Mat();
Cv2.BitwiseNot(wallMask, notWall);
Cv2.BitwiseAnd(exploredMask, notWall, exploredMask);
using var notPlayer = new Mat();
Cv2.BitwiseNot(playerMask, notPlayer);
Cv2.BitwiseAnd(exploredMask, notPlayer, exploredMask);
return exploredMask;
}
private static void FilterSmallComponents(Mat mask, int minArea)
{
if (minArea <= 0) return;
using var labels = new Mat();
using var stats = new Mat();
using var centroids = new Mat();
var numLabels = Cv2.ConnectedComponentsWithStats(mask, labels, stats, centroids);
if (numLabels <= 1) return; // only background
// Clear mask, then re-add only components meeting min area
mask.SetTo(Scalar.Black);
for (var i = 1; i < numLabels; i++)
{
var area = stats.At<int>(i, 4); // CC_STAT_AREA
if (area < minArea) continue;
using var comp = new Mat();
Cv2.Compare(labels, new Scalar(i), comp, CmpType.EQ);
mask.SetTo(new Scalar(255), comp);
}
}
private Mat TemporalSmooth(Mat classified)
{
// Add raw frame to ring buffer (takes ownership)
_frameBuffer.Enqueue(classified);
while (_frameBuffer.Count > _config.TemporalFrameCount)
_frameBuffer.Dequeue().Dispose();
// Not enough frames yet — return as-is
if (_frameBuffer.Count < 2)
return classified.Clone();
var size = classified.Size();
using var wallCount = new Mat(size, MatType.CV_8UC1, Scalar.Black);
using var exploredCount = new Mat(size, MatType.CV_8UC1, Scalar.Black);
foreach (var frame in _frameBuffer)
{
using var isWall = new Mat();
Cv2.Compare(frame, new Scalar((byte)MapCell.Wall), isWall, CmpType.EQ);
Cv2.Add(wallCount, new Scalar(1), wallCount, isWall);
using var isExplored = new Mat();
Cv2.Compare(frame, new Scalar((byte)MapCell.Explored), isExplored, CmpType.EQ);
Cv2.Add(exploredCount, new Scalar(1), exploredCount, isExplored);
}
// Apply vote thresholds
using var wallPass = new Mat();
Cv2.Threshold(wallCount, wallPass,
_config.WallTemporalThreshold - 0.5, 255, ThresholdTypes.Binary);
using var exploredPass = new Mat();
Cv2.Threshold(exploredCount, exploredPass,
_config.ExploredTemporalThreshold - 0.5, 255, ThresholdTypes.Binary);
// Build smoothed result (explored wins over wall)
var result = new Mat(size, MatType.CV_8UC1, Scalar.Black);
result.SetTo(new Scalar((byte)MapCell.Wall), wallPass);
result.SetTo(new Scalar((byte)MapCell.Explored), exploredPass);
return result;
}
/// <summary>
/// Save debug images: raw capture, HSV mask, classified result.
/// Call once to diagnose color ranges.
/// Capture a single frame and return the requested pipeline stage as PNG bytes.
/// </summary>
public byte[]? CaptureStage(MinimapDebugStage stage)
{
var region = _config.CaptureRegion;
using var bgr = _backend.CaptureRegion(region);
if (bgr == null || bgr.Empty()) return null;
if (stage == MinimapDebugStage.Raw) return EncodePng(bgr);
using var hsv = new Mat();
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
if (stage is MinimapDebugStage.Hue or MinimapDebugStage.Saturation or MinimapDebugStage.Value)
{
var channels = Cv2.Split(hsv);
var idx = stage switch { MinimapDebugStage.Hue => 0, MinimapDebugStage.Saturation => 1, _ => 2 };
var result = EncodePng(channels[idx]);
foreach (var c in channels) c.Dispose();
return result;
}
using var satChan = new Mat();
using var valueChan = new Mat();
Cv2.ExtractChannel(hsv, satChan, 1);
Cv2.ExtractChannel(hsv, valueChan, 2);
using var playerMask = new Mat();
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
if (stage == MinimapDebugStage.Player) return EncodePng(playerMask);
using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
if (stage == MinimapDebugStage.Walls) return EncodePng(wallMask);
using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
if (stage == MinimapDebugStage.Explored) return EncodePng(exploredMask);
// Classified
using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
classified.SetTo(new Scalar(104, 64, 31), exploredMask);
classified.SetTo(new Scalar(26, 45, 61), wallMask);
classified.SetTo(new Scalar(0, 165, 255), playerMask);
return EncodePng(classified);
}
private static byte[] EncodePng(Mat mat)
{
Cv2.ImEncode(".png", mat, out var buf);
return buf;
}
/// <summary>
/// Save debug images for tuning thresholds.
/// </summary>
public void SaveDebugCapture(string dir = "debug-minimap")
{
@ -129,29 +263,37 @@ public class MinimapCapture : IDisposable
using var bgr = _backend.CaptureRegion(region);
if (bgr == null || bgr.Empty()) return;
using var masked = new Mat();
Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
using var hsv = new Mat();
Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
using var exploredMask = new Mat();
Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
using var satChan = new Mat();
using var valueChan = new Mat();
Cv2.ExtractChannel(hsv, satChan, 1);
Cv2.ExtractChannel(hsv, valueChan, 2);
using var playerMask = new Mat();
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
// Save raw, masked, explored filter, player filter
Cv2.ImWrite(Path.Combine(dir, "1-raw.png"), bgr);
Cv2.ImWrite(Path.Combine(dir, "2-masked.png"), masked);
Cv2.ImWrite(Path.Combine(dir, "3-explored.png"), exploredMask);
Cv2.ImWrite(Path.Combine(dir, "4-player.png"), playerMask);
using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
// Save HSV channels separately for tuning
// Colorized classified
using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
classified.SetTo(new Scalar(104, 64, 31), exploredMask); // blue
classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown
classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange
Cv2.ImWrite(Path.Combine(dir, "01-raw.png"), bgr);
Cv2.ImWrite(Path.Combine(dir, "02-walls.png"), wallMask);
Cv2.ImWrite(Path.Combine(dir, "03-explored.png"), exploredMask);
Cv2.ImWrite(Path.Combine(dir, "04-player.png"), playerMask);
Cv2.ImWrite(Path.Combine(dir, "05-classified.png"), classified);
// HSV channels
var channels = Cv2.Split(hsv);
Cv2.ImWrite(Path.Combine(dir, "5-hue.png"), channels[0]);
Cv2.ImWrite(Path.Combine(dir, "6-sat.png"), channels[1]);
Cv2.ImWrite(Path.Combine(dir, "7-val.png"), channels[2]);
Cv2.ImWrite(Path.Combine(dir, "06-hue.png"), channels[0]);
Cv2.ImWrite(Path.Combine(dir, "07-sat.png"), channels[1]);
Cv2.ImWrite(Path.Combine(dir, "08-val.png"), channels[2]);
foreach (var c in channels) c.Dispose();
Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir));
@ -170,7 +312,8 @@ public class MinimapCapture : IDisposable
public void Dispose()
{
_circularMask?.Dispose();
_backend.Dispose();
while (_frameBuffer.Count > 0)
_frameBuffer.Dequeue().Dispose();
}
}

View file

@ -58,61 +58,70 @@ public class NavigationExecutor : IDisposable
await _game.ToggleMinimap();
await Helpers.Sleep(300);
var lastMoveTime = long.MinValue;
while (!_stopped)
{
var frameStart = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
try
{
// 1. Capture frame
// 1. Capture + track every frame (~30 fps)
SetState(NavigationState.Capturing);
using var frame = _capture.CaptureFrame();
if (frame == null)
{
Log.Warning("Failed to capture minimap frame");
await Helpers.Sleep(200);
await Helpers.Sleep(_config.CaptureIntervalMs);
continue;
}
// 2. Track position via phase correlation
SetState(NavigationState.Processing);
var pos = _tracker.UpdatePosition(frame.GrayMat);
// 3. Stitch into world map
_worldMap.StitchFrame(frame.ClassifiedMat, pos);
// 4. Check if stuck
if (_tracker.IsStuck)
// 2. Movement decisions at slower rate
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
if (now - lastMoveTime >= _config.MovementWaitMs)
{
SetState(NavigationState.Stuck);
Log.Information("Stuck detected, clicking random direction");
await ClickRandomDirection();
await Helpers.Sleep(_config.MovementWaitMs);
continue;
lastMoveTime = now;
if (_tracker.IsStuck)
{
SetState(NavigationState.Stuck);
Log.Information("Stuck detected, clicking random direction");
await ClickRandomDirection();
}
else
{
SetState(NavigationState.Planning);
var direction = _worldMap.FindNearestUnexplored(pos);
if (direction == null)
{
Log.Information("Map fully explored");
SetState(NavigationState.Completed);
break;
}
SetState(NavigationState.Moving);
await ClickToMove(direction.Value.dirX, direction.Value.dirY);
}
}
// 5. Find best exploration direction
SetState(NavigationState.Planning);
var direction = _worldMap.FindNearestUnexplored(pos);
if (direction == null)
{
Log.Information("Map fully explored");
SetState(NavigationState.Completed);
break;
}
// 6. Click to move in that direction
SetState(NavigationState.Moving);
await ClickToMove(direction.Value.dirX, direction.Value.dirY);
// 7. Wait for character to walk
await Helpers.Sleep(_config.MovementWaitMs);
}
catch (Exception ex)
{
Log.Error(ex, "Error in explore loop");
SetState(NavigationState.Failed);
await Helpers.Sleep(1000);
continue;
}
// Sleep remainder of frame interval
var elapsed = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - frameStart;
var sleepMs = _config.CaptureIntervalMs - (int)elapsed;
if (sleepMs > 0)
await Helpers.Sleep(sleepMs);
}
if (_state != NavigationState.Completed)
@ -149,16 +158,20 @@ public class NavigationExecutor : IDisposable
/// <summary>
/// Capture one frame, track position, stitch into world map.
/// Returns viewport PNG bytes, or null on failure.
/// Returns PNG bytes for the requested debug stage (or world map viewport by default).
/// </summary>
public byte[]? ProcessFrame()
public byte[]? ProcessFrame(MinimapDebugStage stage = MinimapDebugStage.WorldMap)
{
using var frame = _capture.CaptureFrame();
if (frame == null) return null;
var pos = _tracker.UpdatePosition(frame.GrayMat);
_worldMap.StitchFrame(frame.ClassifiedMat, pos);
return _worldMap.GetViewportSnapshot(pos);
if (stage == MinimapDebugStage.WorldMap)
return _worldMap.GetViewportSnapshot(pos);
return _capture.CaptureStage(stage);
}
public void SaveDebugCapture() => _capture.SaveDebugCapture();

View file

@ -24,6 +24,19 @@ public enum MapCell : byte
Wall = 2
}
public enum MinimapDebugStage
{
WorldMap,
Raw,
Walls,
Explored,
Player,
Classified,
Hue,
Saturation,
Value
}
public record MinimapFrame(
Mat GrayMat,
Mat ClassifiedMat,
@ -52,20 +65,27 @@ public class MinimapConfig
public int CaptureSize { get; set; } = 300;
// Fog-of-war circle radius within the captured frame
public int FogRadius { get; set; } = 120;
// HSV range for explored areas (purple/violet minimap lines)
// OpenCV H: 0-180, purple ≈ 120-170
public Scalar ExploredLoHSV { get; set; } = new(120, 40, 40);
public Scalar ExploredHiHSV { get; set; } = new(170, 255, 255);
// HSV range for player marker (orange X)
public Scalar PlayerLoHSV { get; set; } = new(5, 80, 80);
public Scalar PlayerHiHSV { get; set; } = new(25, 255, 255);
// HSV range for walls (dark pixels) — set very low to avoid game-world noise
public int WallMaxValue { get; set; } = 20;
// Wall detection: bright OR saturated pixels (minimap structure lines, icons)
public int WallMinValue { get; set; } = 200;
public int WallMinSat { get; set; } = 150;
// Floor detection: minimum V to distinguish explored floor from fog-of-war
public int FloorMinValue { get; set; } = 15;
// Connected components: minimum area to keep (kills speckle)
public int WallMinArea { get; set; } = 30;
// Temporal smoothing: majority vote over ring buffer
public int TemporalFrameCount { get; set; } = 5;
public int WallTemporalThreshold { get; set; } = 3;
public int ExploredTemporalThreshold { get; set; } = 2;
// Capture rate (~30 fps)
public int CaptureIntervalMs { get; set; } = 33;
// Movement
public int ClickRadius { get; set; } = 100;

View file

@ -32,13 +32,23 @@ public class PositionTracker : IDisposable
return Position;
}
// Convert to float64 for phase correlation
// Convert to float64
using var prev64 = new Mat();
using var curr64 = new Mat();
_prevGray.ConvertTo(prev64, MatType.CV_64F);
currentGray.ConvertTo(curr64, MatType.CV_64F);
var shift = Cv2.PhaseCorrelate(prev64, curr64, _hanningWindow, out var confidence);
// High-pass filter: removes slow lighting changes, keeps edges/structure
using var prevBlur = new Mat();
using var currBlur = new Mat();
Cv2.GaussianBlur(prev64, prevBlur, new OpenCvSharp.Size(21, 21), 0);
Cv2.GaussianBlur(curr64, currBlur, new OpenCvSharp.Size(21, 21), 0);
using var prevHp = new Mat();
using var currHp = new Mat();
Cv2.Subtract(prev64, prevBlur, prevHp);
Cv2.Subtract(curr64, currBlur, currHp);
var shift = Cv2.PhaseCorrelate(prevHp, currHp, _hanningWindow, out var confidence);
if (confidence < _config.ConfidenceThreshold)
{

View file

@ -63,7 +63,7 @@ public class WorldMap : IDisposable
var bestAngle = double.NaN;
var bestScore = 0;
const int sectorCount = 16;
var fogRadius = _config.FogRadius;
var fogRadius = _config.CaptureSize / 2;
for (var sector = 0; sector < sectorCount; sector++)
{

View file

@ -51,6 +51,9 @@ public partial class MainWindowViewModel : ObservableObject
[ObservableProperty] private Bitmap? _inventoryImage;
[ObservableProperty] private Bitmap? _minimapImage;
[ObservableProperty] private string _navigationStateText = "";
[ObservableProperty] private MinimapDebugStage _selectedMinimapStage = MinimapDebugStage.WorldMap;
public static MinimapDebugStage[] MinimapStages { get; } = Enum.GetValues<MinimapDebugStage>();
[ObservableProperty] private string _newUrl = "";
[ObservableProperty] private string _newLinkName = "";
@ -198,7 +201,7 @@ public partial class MainWindowViewModel : ObservableObject
f12WasDown = f12Down;
// Minimap capture + display
var bytes = _bot.Navigation.ProcessFrame();
var bytes = _bot.Navigation.ProcessFrame(SelectedMinimapStage);
if (bytes != null)
{
var bmp = new Bitmap(new MemoryStream(bytes));

View file

@ -113,10 +113,15 @@
<DockPanel>
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,0,0,4">
<TextBlock Text="MINIMAP" FontSize="11" FontWeight="SemiBold"
Foreground="#8b949e" />
Foreground="#8b949e" VerticalAlignment="Center" />
<TextBlock Text="{Binding NavigationStateText}"
FontSize="11" Foreground="#58a6ff" Margin="8,0,0,0" />
FontSize="11" Foreground="#58a6ff" Margin="8,0,0,0"
VerticalAlignment="Center" />
</StackPanel>
<ComboBox DockPanel.Dock="Top"
ItemsSource="{x:Static vm:MainWindowViewModel.MinimapStages}"
SelectedItem="{Binding SelectedMinimapStage}"
FontSize="11" Margin="0,0,0,4" />
<Grid>
<Image Source="{Binding MinimapImage}" Stretch="Uniform"
RenderOptions.BitmapInterpolationMode="None" />