minimap work
This commit is contained in:
parent
802f1030d5
commit
ba8626b470
7 changed files with 312 additions and 118 deletions
|
|
@ -1,7 +1,6 @@
|
||||||
using OpenCvSharp;
|
using OpenCvSharp;
|
||||||
using Serilog;
|
using Serilog;
|
||||||
using Region = Poe2Trade.Core.Region;
|
using Region = Poe2Trade.Core.Region;
|
||||||
using Point = OpenCvSharp.Point;
|
|
||||||
using Size = OpenCvSharp.Size;
|
using Size = OpenCvSharp.Size;
|
||||||
|
|
||||||
namespace Poe2Trade.Navigation;
|
namespace Poe2Trade.Navigation;
|
||||||
|
|
@ -10,13 +9,12 @@ public class MinimapCapture : IDisposable
|
||||||
{
|
{
|
||||||
private readonly MinimapConfig _config;
|
private readonly MinimapConfig _config;
|
||||||
private readonly IScreenCapture _backend;
|
private readonly IScreenCapture _backend;
|
||||||
private Mat? _circularMask;
|
private readonly Queue<Mat> _frameBuffer = new();
|
||||||
|
|
||||||
public MinimapCapture(MinimapConfig config)
|
public MinimapCapture(MinimapConfig config)
|
||||||
{
|
{
|
||||||
_config = config;
|
_config = config;
|
||||||
_backend = CreateBackend();
|
_backend = CreateBackend();
|
||||||
BuildCircularMask();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static IScreenCapture CreateBackend()
|
private static IScreenCapture CreateBackend()
|
||||||
|
|
@ -48,14 +46,6 @@ public class MinimapCapture : IDisposable
|
||||||
return new GdiCapture();
|
return new GdiCapture();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void BuildCircularMask()
|
|
||||||
{
|
|
||||||
var size = _config.CaptureSize;
|
|
||||||
_circularMask = new Mat(size, size, MatType.CV_8UC1, Scalar.Black);
|
|
||||||
var center = new Point(size / 2, size / 2);
|
|
||||||
Cv2.Circle(_circularMask, center, _config.FogRadius, Scalar.White, -1);
|
|
||||||
}
|
|
||||||
|
|
||||||
public MinimapFrame? CaptureFrame()
|
public MinimapFrame? CaptureFrame()
|
||||||
{
|
{
|
||||||
var region = _config.CaptureRegion;
|
var region = _config.CaptureRegion;
|
||||||
|
|
@ -64,63 +54,207 @@ public class MinimapCapture : IDisposable
|
||||||
if (bgr == null || bgr.Empty())
|
if (bgr == null || bgr.Empty())
|
||||||
return null;
|
return null;
|
||||||
|
|
||||||
// Apply circular mask to ignore area outside fog-of-war circle
|
// --- 1. HSV + extract S/V channels ---
|
||||||
using var masked = new Mat();
|
|
||||||
Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
|
|
||||||
|
|
||||||
// Convert to HSV
|
|
||||||
using var hsv = new Mat();
|
using var hsv = new Mat();
|
||||||
Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
|
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
||||||
|
using var satChan = new Mat();
|
||||||
// Classify explored areas
|
|
||||||
using var exploredMask = new Mat();
|
|
||||||
Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
|
|
||||||
|
|
||||||
// Classify walls: dark pixels within the circular mask
|
|
||||||
using var valueChan = new Mat();
|
using var valueChan = new Mat();
|
||||||
Cv2.ExtractChannel(hsv, valueChan, 2); // V channel
|
Cv2.ExtractChannel(hsv, satChan, 1); // S
|
||||||
using var darkMask = new Mat();
|
Cv2.ExtractChannel(hsv, valueChan, 2); // V
|
||||||
Cv2.Threshold(valueChan, darkMask, _config.WallMaxValue, 255, ThresholdTypes.BinaryInv);
|
|
||||||
// Apply morphological close to connect wall fragments
|
|
||||||
using var wallKernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(3, 3));
|
|
||||||
using var wallMask = new Mat();
|
|
||||||
Cv2.MorphologyEx(darkMask, wallMask, MorphTypes.Close, wallKernel);
|
|
||||||
// Only within circular mask
|
|
||||||
Cv2.BitwiseAnd(wallMask, _circularMask!, wallMask);
|
|
||||||
// Don't count explored pixels as wall
|
|
||||||
using var notExplored = new Mat();
|
|
||||||
Cv2.BitwiseNot(exploredMask, notExplored);
|
|
||||||
Cv2.BitwiseAnd(wallMask, notExplored, wallMask);
|
|
||||||
|
|
||||||
// Build classified mat: Unknown=0, Explored=1, Wall=2
|
// --- 2. Player mask (orange marker) ---
|
||||||
var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
|
|
||||||
classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
|
|
||||||
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
|
|
||||||
// Ensure only within circular mask
|
|
||||||
Cv2.BitwiseAnd(classified, _circularMask!, classified);
|
|
||||||
|
|
||||||
// Detect player marker (orange X)
|
|
||||||
using var playerMask = new Mat();
|
using var playerMask = new Mat();
|
||||||
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
||||||
var playerOffset = FindCentroid(playerMask);
|
var playerOffset = FindCentroid(playerMask);
|
||||||
|
|
||||||
// Convert to grayscale for phase correlation
|
// --- 3. Wall mask: bright OR saturated → structure lines ---
|
||||||
var gray = new Mat();
|
using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
|
||||||
Cv2.CvtColor(masked, gray, ColorConversionCodes.BGR2GRAY);
|
|
||||||
// Apply circular mask to gray too
|
// --- 4. Explored mask: brightness above fog-of-war, minus walls/player ---
|
||||||
Cv2.BitwiseAnd(gray, _circularMask!, gray);
|
using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
|
||||||
|
|
||||||
|
// --- 5. Build raw classified mat ---
|
||||||
|
var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black);
|
||||||
|
classified.SetTo(new Scalar((byte)MapCell.Explored), exploredMask);
|
||||||
|
classified.SetTo(new Scalar((byte)MapCell.Wall), wallMask);
|
||||||
|
|
||||||
|
// --- 6. Temporal smoothing: majority vote ---
|
||||||
|
var smoothed = TemporalSmooth(classified); // classified goes into ring buffer
|
||||||
|
|
||||||
|
// --- 7. Gray for phase correlation (walls zeroed out — structural noise) ---
|
||||||
|
var grayForCorr = new Mat();
|
||||||
|
Cv2.CvtColor(bgr, grayForCorr, ColorConversionCodes.BGR2GRAY);
|
||||||
|
grayForCorr.SetTo(Scalar.Black, wallMask);
|
||||||
|
|
||||||
return new MinimapFrame(
|
return new MinimapFrame(
|
||||||
GrayMat: gray,
|
GrayMat: grayForCorr,
|
||||||
ClassifiedMat: classified,
|
ClassifiedMat: smoothed,
|
||||||
PlayerOffset: playerOffset,
|
PlayerOffset: playerOffset,
|
||||||
Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
|
Timestamp: DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Mat BuildWallMask(Mat satChan, Mat valueChan, Mat playerMask)
|
||||||
|
{
|
||||||
|
// Wall = bright OR saturated pixels (minimap structure lines, icons)
|
||||||
|
using var highV = new Mat();
|
||||||
|
Cv2.Threshold(valueChan, highV, _config.WallMinValue, 255, ThresholdTypes.Binary);
|
||||||
|
using var highS = new Mat();
|
||||||
|
Cv2.Threshold(satChan, highS, _config.WallMinSat, 255, ThresholdTypes.Binary);
|
||||||
|
|
||||||
|
var wallMask = new Mat();
|
||||||
|
Cv2.BitwiseOr(highV, highS, wallMask);
|
||||||
|
|
||||||
|
// Dilate to connect wall line fragments
|
||||||
|
using var kernel = Cv2.GetStructuringElement(MorphShapes.Ellipse, new Size(3, 3));
|
||||||
|
Cv2.Dilate(wallMask, wallMask, kernel);
|
||||||
|
|
||||||
|
// Subtract player (bright/saturated marker would otherwise be classified as wall)
|
||||||
|
using var notPlayer = new Mat();
|
||||||
|
Cv2.BitwiseNot(playerMask, notPlayer);
|
||||||
|
Cv2.BitwiseAnd(wallMask, notPlayer, wallMask);
|
||||||
|
|
||||||
|
FilterSmallComponents(wallMask, _config.WallMinArea);
|
||||||
|
return wallMask;
|
||||||
|
}
|
||||||
|
|
||||||
|
private Mat BuildExploredMask(Mat valueChan, Mat wallMask, Mat playerMask)
|
||||||
|
{
|
||||||
|
// Explored = any pixel above fog-of-war darkness, minus walls and player
|
||||||
|
var exploredMask = new Mat();
|
||||||
|
Cv2.Threshold(valueChan, exploredMask, _config.FloorMinValue, 255, ThresholdTypes.Binary);
|
||||||
|
|
||||||
|
using var notWall = new Mat();
|
||||||
|
Cv2.BitwiseNot(wallMask, notWall);
|
||||||
|
Cv2.BitwiseAnd(exploredMask, notWall, exploredMask);
|
||||||
|
|
||||||
|
using var notPlayer = new Mat();
|
||||||
|
Cv2.BitwiseNot(playerMask, notPlayer);
|
||||||
|
Cv2.BitwiseAnd(exploredMask, notPlayer, exploredMask);
|
||||||
|
|
||||||
|
return exploredMask;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void FilterSmallComponents(Mat mask, int minArea)
|
||||||
|
{
|
||||||
|
if (minArea <= 0) return;
|
||||||
|
|
||||||
|
using var labels = new Mat();
|
||||||
|
using var stats = new Mat();
|
||||||
|
using var centroids = new Mat();
|
||||||
|
var numLabels = Cv2.ConnectedComponentsWithStats(mask, labels, stats, centroids);
|
||||||
|
|
||||||
|
if (numLabels <= 1) return; // only background
|
||||||
|
|
||||||
|
// Clear mask, then re-add only components meeting min area
|
||||||
|
mask.SetTo(Scalar.Black);
|
||||||
|
for (var i = 1; i < numLabels; i++)
|
||||||
|
{
|
||||||
|
var area = stats.At<int>(i, 4); // CC_STAT_AREA
|
||||||
|
if (area < minArea) continue;
|
||||||
|
|
||||||
|
using var comp = new Mat();
|
||||||
|
Cv2.Compare(labels, new Scalar(i), comp, CmpType.EQ);
|
||||||
|
mask.SetTo(new Scalar(255), comp);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private Mat TemporalSmooth(Mat classified)
|
||||||
|
{
|
||||||
|
// Add raw frame to ring buffer (takes ownership)
|
||||||
|
_frameBuffer.Enqueue(classified);
|
||||||
|
while (_frameBuffer.Count > _config.TemporalFrameCount)
|
||||||
|
_frameBuffer.Dequeue().Dispose();
|
||||||
|
|
||||||
|
// Not enough frames yet — return as-is
|
||||||
|
if (_frameBuffer.Count < 2)
|
||||||
|
return classified.Clone();
|
||||||
|
|
||||||
|
var size = classified.Size();
|
||||||
|
using var wallCount = new Mat(size, MatType.CV_8UC1, Scalar.Black);
|
||||||
|
using var exploredCount = new Mat(size, MatType.CV_8UC1, Scalar.Black);
|
||||||
|
|
||||||
|
foreach (var frame in _frameBuffer)
|
||||||
|
{
|
||||||
|
using var isWall = new Mat();
|
||||||
|
Cv2.Compare(frame, new Scalar((byte)MapCell.Wall), isWall, CmpType.EQ);
|
||||||
|
Cv2.Add(wallCount, new Scalar(1), wallCount, isWall);
|
||||||
|
|
||||||
|
using var isExplored = new Mat();
|
||||||
|
Cv2.Compare(frame, new Scalar((byte)MapCell.Explored), isExplored, CmpType.EQ);
|
||||||
|
Cv2.Add(exploredCount, new Scalar(1), exploredCount, isExplored);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply vote thresholds
|
||||||
|
using var wallPass = new Mat();
|
||||||
|
Cv2.Threshold(wallCount, wallPass,
|
||||||
|
_config.WallTemporalThreshold - 0.5, 255, ThresholdTypes.Binary);
|
||||||
|
using var exploredPass = new Mat();
|
||||||
|
Cv2.Threshold(exploredCount, exploredPass,
|
||||||
|
_config.ExploredTemporalThreshold - 0.5, 255, ThresholdTypes.Binary);
|
||||||
|
|
||||||
|
// Build smoothed result (explored wins over wall)
|
||||||
|
var result = new Mat(size, MatType.CV_8UC1, Scalar.Black);
|
||||||
|
result.SetTo(new Scalar((byte)MapCell.Wall), wallPass);
|
||||||
|
result.SetTo(new Scalar((byte)MapCell.Explored), exploredPass);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Save debug images: raw capture, HSV mask, classified result.
|
/// Capture a single frame and return the requested pipeline stage as PNG bytes.
|
||||||
/// Call once to diagnose color ranges.
|
/// </summary>
|
||||||
|
public byte[]? CaptureStage(MinimapDebugStage stage)
|
||||||
|
{
|
||||||
|
var region = _config.CaptureRegion;
|
||||||
|
using var bgr = _backend.CaptureRegion(region);
|
||||||
|
if (bgr == null || bgr.Empty()) return null;
|
||||||
|
|
||||||
|
if (stage == MinimapDebugStage.Raw) return EncodePng(bgr);
|
||||||
|
|
||||||
|
using var hsv = new Mat();
|
||||||
|
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
||||||
|
|
||||||
|
if (stage is MinimapDebugStage.Hue or MinimapDebugStage.Saturation or MinimapDebugStage.Value)
|
||||||
|
{
|
||||||
|
var channels = Cv2.Split(hsv);
|
||||||
|
var idx = stage switch { MinimapDebugStage.Hue => 0, MinimapDebugStage.Saturation => 1, _ => 2 };
|
||||||
|
var result = EncodePng(channels[idx]);
|
||||||
|
foreach (var c in channels) c.Dispose();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
using var satChan = new Mat();
|
||||||
|
using var valueChan = new Mat();
|
||||||
|
Cv2.ExtractChannel(hsv, satChan, 1);
|
||||||
|
Cv2.ExtractChannel(hsv, valueChan, 2);
|
||||||
|
|
||||||
|
using var playerMask = new Mat();
|
||||||
|
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
||||||
|
if (stage == MinimapDebugStage.Player) return EncodePng(playerMask);
|
||||||
|
|
||||||
|
using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
|
||||||
|
if (stage == MinimapDebugStage.Walls) return EncodePng(wallMask);
|
||||||
|
|
||||||
|
using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
|
||||||
|
if (stage == MinimapDebugStage.Explored) return EncodePng(exploredMask);
|
||||||
|
|
||||||
|
// Classified
|
||||||
|
using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
|
||||||
|
classified.SetTo(new Scalar(104, 64, 31), exploredMask);
|
||||||
|
classified.SetTo(new Scalar(26, 45, 61), wallMask);
|
||||||
|
classified.SetTo(new Scalar(0, 165, 255), playerMask);
|
||||||
|
return EncodePng(classified);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static byte[] EncodePng(Mat mat)
|
||||||
|
{
|
||||||
|
Cv2.ImEncode(".png", mat, out var buf);
|
||||||
|
return buf;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Save debug images for tuning thresholds.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public void SaveDebugCapture(string dir = "debug-minimap")
|
public void SaveDebugCapture(string dir = "debug-minimap")
|
||||||
{
|
{
|
||||||
|
|
@ -129,29 +263,37 @@ public class MinimapCapture : IDisposable
|
||||||
using var bgr = _backend.CaptureRegion(region);
|
using var bgr = _backend.CaptureRegion(region);
|
||||||
if (bgr == null || bgr.Empty()) return;
|
if (bgr == null || bgr.Empty()) return;
|
||||||
|
|
||||||
using var masked = new Mat();
|
|
||||||
Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!);
|
|
||||||
|
|
||||||
using var hsv = new Mat();
|
using var hsv = new Mat();
|
||||||
Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV);
|
Cv2.CvtColor(bgr, hsv, ColorConversionCodes.BGR2HSV);
|
||||||
|
|
||||||
using var exploredMask = new Mat();
|
using var satChan = new Mat();
|
||||||
Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask);
|
using var valueChan = new Mat();
|
||||||
|
Cv2.ExtractChannel(hsv, satChan, 1);
|
||||||
|
Cv2.ExtractChannel(hsv, valueChan, 2);
|
||||||
|
|
||||||
using var playerMask = new Mat();
|
using var playerMask = new Mat();
|
||||||
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask);
|
||||||
|
|
||||||
// Save raw, masked, explored filter, player filter
|
using var wallMask = BuildWallMask(satChan, valueChan, playerMask);
|
||||||
Cv2.ImWrite(Path.Combine(dir, "1-raw.png"), bgr);
|
using var exploredMask = BuildExploredMask(valueChan, wallMask, playerMask);
|
||||||
Cv2.ImWrite(Path.Combine(dir, "2-masked.png"), masked);
|
|
||||||
Cv2.ImWrite(Path.Combine(dir, "3-explored.png"), exploredMask);
|
|
||||||
Cv2.ImWrite(Path.Combine(dir, "4-player.png"), playerMask);
|
|
||||||
|
|
||||||
// Save HSV channels separately for tuning
|
// Colorized classified
|
||||||
|
using var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC3, Scalar.Black);
|
||||||
|
classified.SetTo(new Scalar(104, 64, 31), exploredMask); // blue
|
||||||
|
classified.SetTo(new Scalar(26, 45, 61), wallMask); // brown
|
||||||
|
classified.SetTo(new Scalar(0, 165, 255), playerMask); // orange
|
||||||
|
|
||||||
|
Cv2.ImWrite(Path.Combine(dir, "01-raw.png"), bgr);
|
||||||
|
Cv2.ImWrite(Path.Combine(dir, "02-walls.png"), wallMask);
|
||||||
|
Cv2.ImWrite(Path.Combine(dir, "03-explored.png"), exploredMask);
|
||||||
|
Cv2.ImWrite(Path.Combine(dir, "04-player.png"), playerMask);
|
||||||
|
Cv2.ImWrite(Path.Combine(dir, "05-classified.png"), classified);
|
||||||
|
|
||||||
|
// HSV channels
|
||||||
var channels = Cv2.Split(hsv);
|
var channels = Cv2.Split(hsv);
|
||||||
Cv2.ImWrite(Path.Combine(dir, "5-hue.png"), channels[0]);
|
Cv2.ImWrite(Path.Combine(dir, "06-hue.png"), channels[0]);
|
||||||
Cv2.ImWrite(Path.Combine(dir, "6-sat.png"), channels[1]);
|
Cv2.ImWrite(Path.Combine(dir, "07-sat.png"), channels[1]);
|
||||||
Cv2.ImWrite(Path.Combine(dir, "7-val.png"), channels[2]);
|
Cv2.ImWrite(Path.Combine(dir, "08-val.png"), channels[2]);
|
||||||
foreach (var c in channels) c.Dispose();
|
foreach (var c in channels) c.Dispose();
|
||||||
|
|
||||||
Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir));
|
Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir));
|
||||||
|
|
@ -170,7 +312,8 @@ public class MinimapCapture : IDisposable
|
||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
_circularMask?.Dispose();
|
|
||||||
_backend.Dispose();
|
_backend.Dispose();
|
||||||
|
while (_frameBuffer.Count > 0)
|
||||||
|
_frameBuffer.Dequeue().Dispose();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -58,38 +58,42 @@ public class NavigationExecutor : IDisposable
|
||||||
await _game.ToggleMinimap();
|
await _game.ToggleMinimap();
|
||||||
await Helpers.Sleep(300);
|
await Helpers.Sleep(300);
|
||||||
|
|
||||||
|
var lastMoveTime = long.MinValue;
|
||||||
|
|
||||||
while (!_stopped)
|
while (!_stopped)
|
||||||
{
|
{
|
||||||
|
var frameStart = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// 1. Capture frame
|
// 1. Capture + track every frame (~30 fps)
|
||||||
SetState(NavigationState.Capturing);
|
SetState(NavigationState.Capturing);
|
||||||
using var frame = _capture.CaptureFrame();
|
using var frame = _capture.CaptureFrame();
|
||||||
if (frame == null)
|
if (frame == null)
|
||||||
{
|
{
|
||||||
Log.Warning("Failed to capture minimap frame");
|
Log.Warning("Failed to capture minimap frame");
|
||||||
await Helpers.Sleep(200);
|
await Helpers.Sleep(_config.CaptureIntervalMs);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Track position via phase correlation
|
|
||||||
SetState(NavigationState.Processing);
|
SetState(NavigationState.Processing);
|
||||||
var pos = _tracker.UpdatePosition(frame.GrayMat);
|
var pos = _tracker.UpdatePosition(frame.GrayMat);
|
||||||
|
|
||||||
// 3. Stitch into world map
|
|
||||||
_worldMap.StitchFrame(frame.ClassifiedMat, pos);
|
_worldMap.StitchFrame(frame.ClassifiedMat, pos);
|
||||||
|
|
||||||
// 4. Check if stuck
|
// 2. Movement decisions at slower rate
|
||||||
|
var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
|
||||||
|
if (now - lastMoveTime >= _config.MovementWaitMs)
|
||||||
|
{
|
||||||
|
lastMoveTime = now;
|
||||||
|
|
||||||
if (_tracker.IsStuck)
|
if (_tracker.IsStuck)
|
||||||
{
|
{
|
||||||
SetState(NavigationState.Stuck);
|
SetState(NavigationState.Stuck);
|
||||||
Log.Information("Stuck detected, clicking random direction");
|
Log.Information("Stuck detected, clicking random direction");
|
||||||
await ClickRandomDirection();
|
await ClickRandomDirection();
|
||||||
await Helpers.Sleep(_config.MovementWaitMs);
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
else
|
||||||
// 5. Find best exploration direction
|
{
|
||||||
SetState(NavigationState.Planning);
|
SetState(NavigationState.Planning);
|
||||||
var direction = _worldMap.FindNearestUnexplored(pos);
|
var direction = _worldMap.FindNearestUnexplored(pos);
|
||||||
|
|
||||||
|
|
@ -100,19 +104,24 @@ public class NavigationExecutor : IDisposable
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. Click to move in that direction
|
|
||||||
SetState(NavigationState.Moving);
|
SetState(NavigationState.Moving);
|
||||||
await ClickToMove(direction.Value.dirX, direction.Value.dirY);
|
await ClickToMove(direction.Value.dirX, direction.Value.dirY);
|
||||||
|
}
|
||||||
// 7. Wait for character to walk
|
}
|
||||||
await Helpers.Sleep(_config.MovementWaitMs);
|
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Log.Error(ex, "Error in explore loop");
|
Log.Error(ex, "Error in explore loop");
|
||||||
SetState(NavigationState.Failed);
|
SetState(NavigationState.Failed);
|
||||||
await Helpers.Sleep(1000);
|
await Helpers.Sleep(1000);
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sleep remainder of frame interval
|
||||||
|
var elapsed = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds() - frameStart;
|
||||||
|
var sleepMs = _config.CaptureIntervalMs - (int)elapsed;
|
||||||
|
if (sleepMs > 0)
|
||||||
|
await Helpers.Sleep(sleepMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (_state != NavigationState.Completed)
|
if (_state != NavigationState.Completed)
|
||||||
|
|
@ -149,16 +158,20 @@ public class NavigationExecutor : IDisposable
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Capture one frame, track position, stitch into world map.
|
/// Capture one frame, track position, stitch into world map.
|
||||||
/// Returns viewport PNG bytes, or null on failure.
|
/// Returns PNG bytes for the requested debug stage (or world map viewport by default).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public byte[]? ProcessFrame()
|
public byte[]? ProcessFrame(MinimapDebugStage stage = MinimapDebugStage.WorldMap)
|
||||||
{
|
{
|
||||||
using var frame = _capture.CaptureFrame();
|
using var frame = _capture.CaptureFrame();
|
||||||
if (frame == null) return null;
|
if (frame == null) return null;
|
||||||
|
|
||||||
var pos = _tracker.UpdatePosition(frame.GrayMat);
|
var pos = _tracker.UpdatePosition(frame.GrayMat);
|
||||||
_worldMap.StitchFrame(frame.ClassifiedMat, pos);
|
_worldMap.StitchFrame(frame.ClassifiedMat, pos);
|
||||||
|
|
||||||
|
if (stage == MinimapDebugStage.WorldMap)
|
||||||
return _worldMap.GetViewportSnapshot(pos);
|
return _worldMap.GetViewportSnapshot(pos);
|
||||||
|
|
||||||
|
return _capture.CaptureStage(stage);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void SaveDebugCapture() => _capture.SaveDebugCapture();
|
public void SaveDebugCapture() => _capture.SaveDebugCapture();
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,19 @@ public enum MapCell : byte
|
||||||
Wall = 2
|
Wall = 2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public enum MinimapDebugStage
|
||||||
|
{
|
||||||
|
WorldMap,
|
||||||
|
Raw,
|
||||||
|
Walls,
|
||||||
|
Explored,
|
||||||
|
Player,
|
||||||
|
Classified,
|
||||||
|
Hue,
|
||||||
|
Saturation,
|
||||||
|
Value
|
||||||
|
}
|
||||||
|
|
||||||
public record MinimapFrame(
|
public record MinimapFrame(
|
||||||
Mat GrayMat,
|
Mat GrayMat,
|
||||||
Mat ClassifiedMat,
|
Mat ClassifiedMat,
|
||||||
|
|
@ -52,20 +65,27 @@ public class MinimapConfig
|
||||||
|
|
||||||
public int CaptureSize { get; set; } = 300;
|
public int CaptureSize { get; set; } = 300;
|
||||||
|
|
||||||
// Fog-of-war circle radius within the captured frame
|
|
||||||
public int FogRadius { get; set; } = 120;
|
|
||||||
|
|
||||||
// HSV range for explored areas (purple/violet minimap lines)
|
|
||||||
// OpenCV H: 0-180, purple ≈ 120-170
|
|
||||||
public Scalar ExploredLoHSV { get; set; } = new(120, 40, 40);
|
|
||||||
public Scalar ExploredHiHSV { get; set; } = new(170, 255, 255);
|
|
||||||
|
|
||||||
// HSV range for player marker (orange X)
|
// HSV range for player marker (orange X)
|
||||||
public Scalar PlayerLoHSV { get; set; } = new(5, 80, 80);
|
public Scalar PlayerLoHSV { get; set; } = new(5, 80, 80);
|
||||||
public Scalar PlayerHiHSV { get; set; } = new(25, 255, 255);
|
public Scalar PlayerHiHSV { get; set; } = new(25, 255, 255);
|
||||||
|
|
||||||
// HSV range for walls (dark pixels) — set very low to avoid game-world noise
|
// Wall detection: bright OR saturated pixels (minimap structure lines, icons)
|
||||||
public int WallMaxValue { get; set; } = 20;
|
public int WallMinValue { get; set; } = 200;
|
||||||
|
public int WallMinSat { get; set; } = 150;
|
||||||
|
|
||||||
|
// Floor detection: minimum V to distinguish explored floor from fog-of-war
|
||||||
|
public int FloorMinValue { get; set; } = 15;
|
||||||
|
|
||||||
|
// Connected components: minimum area to keep (kills speckle)
|
||||||
|
public int WallMinArea { get; set; } = 30;
|
||||||
|
|
||||||
|
// Temporal smoothing: majority vote over ring buffer
|
||||||
|
public int TemporalFrameCount { get; set; } = 5;
|
||||||
|
public int WallTemporalThreshold { get; set; } = 3;
|
||||||
|
public int ExploredTemporalThreshold { get; set; } = 2;
|
||||||
|
|
||||||
|
// Capture rate (~30 fps)
|
||||||
|
public int CaptureIntervalMs { get; set; } = 33;
|
||||||
|
|
||||||
// Movement
|
// Movement
|
||||||
public int ClickRadius { get; set; } = 100;
|
public int ClickRadius { get; set; } = 100;
|
||||||
|
|
|
||||||
|
|
@ -32,13 +32,23 @@ public class PositionTracker : IDisposable
|
||||||
return Position;
|
return Position;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert to float64 for phase correlation
|
// Convert to float64
|
||||||
using var prev64 = new Mat();
|
using var prev64 = new Mat();
|
||||||
using var curr64 = new Mat();
|
using var curr64 = new Mat();
|
||||||
_prevGray.ConvertTo(prev64, MatType.CV_64F);
|
_prevGray.ConvertTo(prev64, MatType.CV_64F);
|
||||||
currentGray.ConvertTo(curr64, MatType.CV_64F);
|
currentGray.ConvertTo(curr64, MatType.CV_64F);
|
||||||
|
|
||||||
var shift = Cv2.PhaseCorrelate(prev64, curr64, _hanningWindow, out var confidence);
|
// High-pass filter: removes slow lighting changes, keeps edges/structure
|
||||||
|
using var prevBlur = new Mat();
|
||||||
|
using var currBlur = new Mat();
|
||||||
|
Cv2.GaussianBlur(prev64, prevBlur, new OpenCvSharp.Size(21, 21), 0);
|
||||||
|
Cv2.GaussianBlur(curr64, currBlur, new OpenCvSharp.Size(21, 21), 0);
|
||||||
|
using var prevHp = new Mat();
|
||||||
|
using var currHp = new Mat();
|
||||||
|
Cv2.Subtract(prev64, prevBlur, prevHp);
|
||||||
|
Cv2.Subtract(curr64, currBlur, currHp);
|
||||||
|
|
||||||
|
var shift = Cv2.PhaseCorrelate(prevHp, currHp, _hanningWindow, out var confidence);
|
||||||
|
|
||||||
if (confidence < _config.ConfidenceThreshold)
|
if (confidence < _config.ConfidenceThreshold)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -63,7 +63,7 @@ public class WorldMap : IDisposable
|
||||||
var bestAngle = double.NaN;
|
var bestAngle = double.NaN;
|
||||||
var bestScore = 0;
|
var bestScore = 0;
|
||||||
const int sectorCount = 16;
|
const int sectorCount = 16;
|
||||||
var fogRadius = _config.FogRadius;
|
var fogRadius = _config.CaptureSize / 2;
|
||||||
|
|
||||||
for (var sector = 0; sector < sectorCount; sector++)
|
for (var sector = 0; sector < sectorCount; sector++)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,9 @@ public partial class MainWindowViewModel : ObservableObject
|
||||||
[ObservableProperty] private Bitmap? _inventoryImage;
|
[ObservableProperty] private Bitmap? _inventoryImage;
|
||||||
[ObservableProperty] private Bitmap? _minimapImage;
|
[ObservableProperty] private Bitmap? _minimapImage;
|
||||||
[ObservableProperty] private string _navigationStateText = "";
|
[ObservableProperty] private string _navigationStateText = "";
|
||||||
|
[ObservableProperty] private MinimapDebugStage _selectedMinimapStage = MinimapDebugStage.WorldMap;
|
||||||
|
|
||||||
|
public static MinimapDebugStage[] MinimapStages { get; } = Enum.GetValues<MinimapDebugStage>();
|
||||||
|
|
||||||
[ObservableProperty] private string _newUrl = "";
|
[ObservableProperty] private string _newUrl = "";
|
||||||
[ObservableProperty] private string _newLinkName = "";
|
[ObservableProperty] private string _newLinkName = "";
|
||||||
|
|
@ -198,7 +201,7 @@ public partial class MainWindowViewModel : ObservableObject
|
||||||
f12WasDown = f12Down;
|
f12WasDown = f12Down;
|
||||||
|
|
||||||
// Minimap capture + display
|
// Minimap capture + display
|
||||||
var bytes = _bot.Navigation.ProcessFrame();
|
var bytes = _bot.Navigation.ProcessFrame(SelectedMinimapStage);
|
||||||
if (bytes != null)
|
if (bytes != null)
|
||||||
{
|
{
|
||||||
var bmp = new Bitmap(new MemoryStream(bytes));
|
var bmp = new Bitmap(new MemoryStream(bytes));
|
||||||
|
|
|
||||||
|
|
@ -113,10 +113,15 @@
|
||||||
<DockPanel>
|
<DockPanel>
|
||||||
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,0,0,4">
|
<StackPanel DockPanel.Dock="Top" Orientation="Horizontal" Margin="0,0,0,4">
|
||||||
<TextBlock Text="MINIMAP" FontSize="11" FontWeight="SemiBold"
|
<TextBlock Text="MINIMAP" FontSize="11" FontWeight="SemiBold"
|
||||||
Foreground="#8b949e" />
|
Foreground="#8b949e" VerticalAlignment="Center" />
|
||||||
<TextBlock Text="{Binding NavigationStateText}"
|
<TextBlock Text="{Binding NavigationStateText}"
|
||||||
FontSize="11" Foreground="#58a6ff" Margin="8,0,0,0" />
|
FontSize="11" Foreground="#58a6ff" Margin="8,0,0,0"
|
||||||
|
VerticalAlignment="Center" />
|
||||||
</StackPanel>
|
</StackPanel>
|
||||||
|
<ComboBox DockPanel.Dock="Top"
|
||||||
|
ItemsSource="{x:Static vm:MainWindowViewModel.MinimapStages}"
|
||||||
|
SelectedItem="{Binding SelectedMinimapStage}"
|
||||||
|
FontSize="11" Margin="0,0,0,4" />
|
||||||
<Grid>
|
<Grid>
|
||||||
<Image Source="{Binding MinimapImage}" Stretch="Uniform"
|
<Image Source="{Binding MinimapImage}" Stretch="Uniform"
|
||||||
RenderOptions.BitmapInterpolationMode="None" />
|
RenderOptions.BitmapInterpolationMode="None" />
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue