264 lines
9.8 KiB
C#
264 lines
9.8 KiB
C#
using OpenCvSharp;
|
|
using Automata.Core;
|
|
using Automata.Screen;
|
|
using Serilog;
|
|
|
|
namespace Automata.Navigation;
|
|
|
|
public record CalibrationResult(float BestFactor, double BestConfidence, Dictionary<float, double> AllResults);
|
|
|
|
/// <summary>
|
|
/// Collects atlas frames and tests different perspective factors to find the optimal one.
|
|
/// For each candidate factor, stitches all frames into a mini-canvas and measures seam
|
|
/// quality (pixel difference in overlap regions). The correct factor minimizes seam error
|
|
/// because frames from different scroll directions align properly.
|
|
/// </summary>
|
|
public class PerspectiveCalibrator : IFrameConsumer, IDisposable
|
|
{
|
|
private readonly List<Mat> _frames = new();
|
|
private Mat? _lastKept;
|
|
|
|
private static readonly Region CaptureRegion = new(300, 150, 1960, 1090);
|
|
private const int MaxFrames = 100;
|
|
private const double MovementThreshold = 4.0;
|
|
|
|
public int FramesCollected => _frames.Count;
|
|
public event Action<int>? FrameCollected;
|
|
|
|
public void Process(ScreenFrame frame)
|
|
{
|
|
if (_frames.Count >= MaxFrames) return;
|
|
|
|
using var bgr = frame.CropBgr(CaptureRegion);
|
|
|
|
// Movement check at 1/8 scale
|
|
using var small = new Mat();
|
|
Cv2.Resize(bgr, small, new Size(bgr.Width / 8, bgr.Height / 8));
|
|
|
|
if (_lastKept != null)
|
|
{
|
|
using var diff = new Mat();
|
|
Cv2.Absdiff(small, _lastKept, diff);
|
|
var mean = Cv2.Mean(diff);
|
|
var avgDiff = (mean.Val0 + mean.Val1 + mean.Val2) / 3.0;
|
|
if (avgDiff < MovementThreshold)
|
|
return;
|
|
}
|
|
|
|
_lastKept?.Dispose();
|
|
_lastKept = small.Clone();
|
|
_frames.Add(bgr.Clone());
|
|
FrameCollected?.Invoke(_frames.Count);
|
|
Log.Debug("PerspectiveCalibrator: kept frame {N}/{Max}", _frames.Count, MaxFrames);
|
|
}
|
|
|
|
public CalibrationResult Calibrate()
|
|
{
|
|
if (_frames.Count < 3)
|
|
throw new InvalidOperationException($"Need at least 3 frames, got {_frames.Count}");
|
|
|
|
Log.Information("PerspectiveCalibrator: analyzing {N} frames with seam-error metric", _frames.Count);
|
|
|
|
var results = new Dictionary<float, double>();
|
|
|
|
// Coarse pass: 0.00 to 0.25, step 0.01
|
|
for (int fi = 0; fi <= 25; fi++)
|
|
{
|
|
var f = fi * 0.01f;
|
|
var quality = MeasureFactorQuality(f);
|
|
results[f] = quality;
|
|
Log.Information("Calibrate: factor={F:F2} seamQuality={Q:F6}", f, quality);
|
|
}
|
|
|
|
// Fine pass around the best coarse value
|
|
var coarseBest = results.MaxBy(kv => kv.Value).Key;
|
|
for (int fi = -9; fi <= 9; fi += 2)
|
|
{
|
|
var f = MathF.Round(coarseBest + fi * 0.001f, 3);
|
|
if (f < 0 || f > 0.30f || results.ContainsKey(f)) continue;
|
|
var quality = MeasureFactorQuality(f);
|
|
results[f] = quality;
|
|
Log.Information("Calibrate (fine): factor={F:F3} seamQuality={Q:F6}", f, quality);
|
|
}
|
|
|
|
var best = results.MaxBy(kv => kv.Value);
|
|
Log.Information("PerspectiveCalibrator: BEST factor={F:F3} seamQuality={Q:F6}", best.Key, best.Value);
|
|
|
|
return new CalibrationResult(best.Key, best.Value, results);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Stitches all frames into a mini-canvas using the given perspective factor,
|
|
/// and measures the average seam quality (pixel alignment in overlap regions).
|
|
/// When the factor is correct, overlapping regions from different scroll directions
|
|
/// align perfectly → low pixel difference → high quality score.
|
|
/// </summary>
|
|
private double MeasureFactorQuality(float factor)
|
|
{
|
|
const int scale = 4;
|
|
var w = _frames[0].Width;
|
|
var h = _frames[0].Height;
|
|
var sw = w / scale;
|
|
var sh = h / scale;
|
|
|
|
// Compute warp matrix at reduced resolution
|
|
Mat? warpMatrix = null;
|
|
if (factor > 0.001f)
|
|
{
|
|
var inset = (int)(sw * factor);
|
|
var src = new Point2f[]
|
|
{
|
|
new(inset, 0), new(sw - inset, 0),
|
|
new(sw, sh), new(0, sh),
|
|
};
|
|
var dst = new Point2f[]
|
|
{
|
|
new(0, 0), new(sw, 0),
|
|
new(sw, sh), new(0, sh),
|
|
};
|
|
warpMatrix = Cv2.GetPerspectiveTransform(src, dst);
|
|
}
|
|
|
|
var smallFrames = new List<Mat>();
|
|
try
|
|
{
|
|
// Downscale first, then warp
|
|
foreach (var frame in _frames)
|
|
{
|
|
var small = new Mat();
|
|
Cv2.Resize(frame, small, new Size(sw, sh));
|
|
|
|
if (warpMatrix != null)
|
|
{
|
|
var warped = new Mat();
|
|
Cv2.WarpPerspective(small, warped, warpMatrix, new Size(sw, sh));
|
|
small.Dispose();
|
|
smallFrames.Add(warped);
|
|
}
|
|
else
|
|
{
|
|
smallFrames.Add(small);
|
|
}
|
|
}
|
|
|
|
// Build mini-canvas and measure seam quality
|
|
const int canvasSize = 4000;
|
|
using var canvas = new Mat(canvasSize, canvasSize, MatType.CV_8UC3, Scalar.Black);
|
|
var vx = canvasSize / 2;
|
|
var vy = canvasSize / 2;
|
|
|
|
// Paste first frame
|
|
PasteMini(canvas, canvasSize, smallFrames[0], vx - sw / 2, vy - sh / 2);
|
|
|
|
const int templateW = 50;
|
|
const int templateH = 50;
|
|
const int searchMargin = 80; // at 1/4 scale ≈ 320px full res
|
|
|
|
var seamQualities = new List<double>();
|
|
|
|
for (int i = 1; i < smallFrames.Count; i++)
|
|
{
|
|
var frame = smallFrames[i];
|
|
|
|
// Find position via center template match against canvas
|
|
var tx = (sw - templateW) / 2;
|
|
var ty = (sh - templateH) / 2;
|
|
|
|
var halfW = sw / 2 + searchMargin;
|
|
var halfH = sh / 2 + searchMargin;
|
|
var sx0 = Math.Max(0, vx - halfW);
|
|
var sy0 = Math.Max(0, vy - halfH);
|
|
var sx1 = Math.Min(canvasSize, vx + halfW);
|
|
var sy1 = Math.Min(canvasSize, vy + halfH);
|
|
var sW = sx1 - sx0;
|
|
var sH = sy1 - sy0;
|
|
|
|
if (sW <= templateW || sH <= templateH) continue;
|
|
|
|
using var tmpl = new Mat(frame, new Rect(tx, ty, templateW, templateH));
|
|
using var searchRoi = new Mat(canvas, new Rect(sx0, sy0, sW, sH));
|
|
using var result = new Mat();
|
|
Cv2.MatchTemplate(searchRoi, tmpl, result, TemplateMatchModes.CCoeffNormed);
|
|
Cv2.MinMaxLoc(result, out _, out var maxVal, out _, out var maxLoc);
|
|
|
|
if (maxVal < 0.5) continue;
|
|
|
|
var frameX = sx0 + maxLoc.X - tx;
|
|
var frameY = sy0 + maxLoc.Y - ty;
|
|
var newVx = frameX + sw / 2;
|
|
var newVy = frameY + sh / 2;
|
|
|
|
// Skip near-static frames
|
|
if (Math.Abs(newVx - vx) < 5 && Math.Abs(newVy - vy) < 5) continue;
|
|
|
|
// Measure seam quality: pixel difference in overlap with existing canvas
|
|
var overlapX0 = Math.Max(frameX, 0);
|
|
var overlapY0 = Math.Max(frameY, 0);
|
|
var overlapX1 = Math.Min(frameX + sw, canvasSize);
|
|
var overlapY1 = Math.Min(frameY + sh, canvasSize);
|
|
var overlapW = overlapX1 - overlapX0;
|
|
var overlapH = overlapY1 - overlapY0;
|
|
|
|
if (overlapW > 20 && overlapH > 20)
|
|
{
|
|
var fsx = overlapX0 - frameX;
|
|
var fsy = overlapY0 - frameY;
|
|
|
|
using var canvasOverlap = new Mat(canvas, new Rect(overlapX0, overlapY0, overlapW, overlapH));
|
|
using var frameOverlap = new Mat(frame, new Rect(fsx, fsy, overlapW, overlapH));
|
|
|
|
// Only measure where canvas already has content (non-black)
|
|
using var gray = new Mat();
|
|
Cv2.CvtColor(canvasOverlap, gray, ColorConversionCodes.BGR2GRAY);
|
|
using var mask = new Mat();
|
|
Cv2.Threshold(gray, mask, 5, 255, ThresholdTypes.Binary);
|
|
|
|
var nonZero = Cv2.CountNonZero(mask);
|
|
if (nonZero > 500)
|
|
{
|
|
using var diff = new Mat();
|
|
Cv2.Absdiff(canvasOverlap, frameOverlap, diff);
|
|
var meanDiff = Cv2.Mean(diff, mask);
|
|
var avgDiff = (meanDiff.Val0 + meanDiff.Val1 + meanDiff.Val2) / 3.0;
|
|
seamQualities.Add(1.0 - avgDiff / 255.0);
|
|
}
|
|
}
|
|
|
|
// Paste frame onto canvas for future overlap comparisons
|
|
PasteMini(canvas, canvasSize, frame, frameX, frameY);
|
|
vx = newVx;
|
|
vy = newVy;
|
|
}
|
|
|
|
return seamQualities.Count > 0 ? seamQualities.Average() : 0;
|
|
}
|
|
finally
|
|
{
|
|
foreach (var s in smallFrames) s.Dispose();
|
|
warpMatrix?.Dispose();
|
|
}
|
|
}
|
|
|
|
private static void PasteMini(Mat canvas, int canvasSize, Mat frame, int canvasX, int canvasY)
|
|
{
|
|
var srcX = Math.Max(0, -canvasX);
|
|
var srcY = Math.Max(0, -canvasY);
|
|
var dstX = Math.Max(0, canvasX);
|
|
var dstY = Math.Max(0, canvasY);
|
|
var w = Math.Min(frame.Width - srcX, canvasSize - dstX);
|
|
var h = Math.Min(frame.Height - srcY, canvasSize - dstY);
|
|
|
|
if (w <= 0 || h <= 0) return;
|
|
|
|
using var srcRoi = new Mat(frame, new Rect(srcX, srcY, w, h));
|
|
using var dstRoi = new Mat(canvas, new Rect(dstX, dstY, w, h));
|
|
srcRoi.CopyTo(dstRoi);
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
foreach (var f in _frames) f.Dispose();
|
|
_frames.Clear();
|
|
_lastKept?.Dispose();
|
|
}
|
|
}
|