using OpenCvSharp; using Automata.Core; using Automata.Screen; using Serilog; namespace Automata.Navigation; public record CalibrationResult(float BestFactor, double BestConfidence, Dictionary AllResults); /// /// Collects atlas frames and tests different perspective factors to find the optimal one. /// For each candidate factor, stitches all frames into a mini-canvas and measures seam /// quality (pixel difference in overlap regions). The correct factor minimizes seam error /// because frames from different scroll directions align properly. /// public class PerspectiveCalibrator : IFrameConsumer, IDisposable { private readonly List _frames = new(); private Mat? _lastKept; private static readonly Region CaptureRegion = new(300, 150, 1960, 1090); private const int MaxFrames = 100; private const double MovementThreshold = 4.0; public int FramesCollected => _frames.Count; public event Action? FrameCollected; public void Process(ScreenFrame frame) { if (_frames.Count >= MaxFrames) return; using var bgr = frame.CropBgr(CaptureRegion); // Movement check at 1/8 scale using var small = new Mat(); Cv2.Resize(bgr, small, new Size(bgr.Width / 8, bgr.Height / 8)); if (_lastKept != null) { using var diff = new Mat(); Cv2.Absdiff(small, _lastKept, diff); var mean = Cv2.Mean(diff); var avgDiff = (mean.Val0 + mean.Val1 + mean.Val2) / 3.0; if (avgDiff < MovementThreshold) return; } _lastKept?.Dispose(); _lastKept = small.Clone(); _frames.Add(bgr.Clone()); FrameCollected?.Invoke(_frames.Count); Log.Debug("PerspectiveCalibrator: kept frame {N}/{Max}", _frames.Count, MaxFrames); } public CalibrationResult Calibrate() { if (_frames.Count < 3) throw new InvalidOperationException($"Need at least 3 frames, got {_frames.Count}"); Log.Information("PerspectiveCalibrator: analyzing {N} frames with seam-error metric", _frames.Count); var results = new Dictionary(); // Coarse pass: 0.00 to 0.25, step 0.01 for (int fi = 0; fi <= 25; fi++) { var f = fi * 0.01f; var quality = MeasureFactorQuality(f); results[f] = quality; Log.Information("Calibrate: factor={F:F2} seamQuality={Q:F6}", f, quality); } // Fine pass around the best coarse value var coarseBest = results.MaxBy(kv => kv.Value).Key; for (int fi = -9; fi <= 9; fi += 2) { var f = MathF.Round(coarseBest + fi * 0.001f, 3); if (f < 0 || f > 0.30f || results.ContainsKey(f)) continue; var quality = MeasureFactorQuality(f); results[f] = quality; Log.Information("Calibrate (fine): factor={F:F3} seamQuality={Q:F6}", f, quality); } var best = results.MaxBy(kv => kv.Value); Log.Information("PerspectiveCalibrator: BEST factor={F:F3} seamQuality={Q:F6}", best.Key, best.Value); return new CalibrationResult(best.Key, best.Value, results); } /// /// Stitches all frames into a mini-canvas using the given perspective factor, /// and measures the average seam quality (pixel alignment in overlap regions). /// When the factor is correct, overlapping regions from different scroll directions /// align perfectly → low pixel difference → high quality score. /// private double MeasureFactorQuality(float factor) { const int scale = 4; var w = _frames[0].Width; var h = _frames[0].Height; var sw = w / scale; var sh = h / scale; // Compute warp matrix at reduced resolution Mat? warpMatrix = null; if (factor > 0.001f) { var inset = (int)(sw * factor); var src = new Point2f[] { new(inset, 0), new(sw - inset, 0), new(sw, sh), new(0, sh), }; var dst = new Point2f[] { new(0, 0), new(sw, 0), new(sw, sh), new(0, sh), }; warpMatrix = Cv2.GetPerspectiveTransform(src, dst); } var smallFrames = new List(); try { // Downscale first, then warp foreach (var frame in _frames) { var small = new Mat(); Cv2.Resize(frame, small, new Size(sw, sh)); if (warpMatrix != null) { var warped = new Mat(); Cv2.WarpPerspective(small, warped, warpMatrix, new Size(sw, sh)); small.Dispose(); smallFrames.Add(warped); } else { smallFrames.Add(small); } } // Build mini-canvas and measure seam quality const int canvasSize = 4000; using var canvas = new Mat(canvasSize, canvasSize, MatType.CV_8UC3, Scalar.Black); var vx = canvasSize / 2; var vy = canvasSize / 2; // Paste first frame PasteMini(canvas, canvasSize, smallFrames[0], vx - sw / 2, vy - sh / 2); const int templateW = 50; const int templateH = 50; const int searchMargin = 80; // at 1/4 scale ≈ 320px full res var seamQualities = new List(); for (int i = 1; i < smallFrames.Count; i++) { var frame = smallFrames[i]; // Find position via center template match against canvas var tx = (sw - templateW) / 2; var ty = (sh - templateH) / 2; var halfW = sw / 2 + searchMargin; var halfH = sh / 2 + searchMargin; var sx0 = Math.Max(0, vx - halfW); var sy0 = Math.Max(0, vy - halfH); var sx1 = Math.Min(canvasSize, vx + halfW); var sy1 = Math.Min(canvasSize, vy + halfH); var sW = sx1 - sx0; var sH = sy1 - sy0; if (sW <= templateW || sH <= templateH) continue; using var tmpl = new Mat(frame, new Rect(tx, ty, templateW, templateH)); using var searchRoi = new Mat(canvas, new Rect(sx0, sy0, sW, sH)); using var result = new Mat(); Cv2.MatchTemplate(searchRoi, tmpl, result, TemplateMatchModes.CCoeffNormed); Cv2.MinMaxLoc(result, out _, out var maxVal, out _, out var maxLoc); if (maxVal < 0.5) continue; var frameX = sx0 + maxLoc.X - tx; var frameY = sy0 + maxLoc.Y - ty; var newVx = frameX + sw / 2; var newVy = frameY + sh / 2; // Skip near-static frames if (Math.Abs(newVx - vx) < 5 && Math.Abs(newVy - vy) < 5) continue; // Measure seam quality: pixel difference in overlap with existing canvas var overlapX0 = Math.Max(frameX, 0); var overlapY0 = Math.Max(frameY, 0); var overlapX1 = Math.Min(frameX + sw, canvasSize); var overlapY1 = Math.Min(frameY + sh, canvasSize); var overlapW = overlapX1 - overlapX0; var overlapH = overlapY1 - overlapY0; if (overlapW > 20 && overlapH > 20) { var fsx = overlapX0 - frameX; var fsy = overlapY0 - frameY; using var canvasOverlap = new Mat(canvas, new Rect(overlapX0, overlapY0, overlapW, overlapH)); using var frameOverlap = new Mat(frame, new Rect(fsx, fsy, overlapW, overlapH)); // Only measure where canvas already has content (non-black) using var gray = new Mat(); Cv2.CvtColor(canvasOverlap, gray, ColorConversionCodes.BGR2GRAY); using var mask = new Mat(); Cv2.Threshold(gray, mask, 5, 255, ThresholdTypes.Binary); var nonZero = Cv2.CountNonZero(mask); if (nonZero > 500) { using var diff = new Mat(); Cv2.Absdiff(canvasOverlap, frameOverlap, diff); var meanDiff = Cv2.Mean(diff, mask); var avgDiff = (meanDiff.Val0 + meanDiff.Val1 + meanDiff.Val2) / 3.0; seamQualities.Add(1.0 - avgDiff / 255.0); } } // Paste frame onto canvas for future overlap comparisons PasteMini(canvas, canvasSize, frame, frameX, frameY); vx = newVx; vy = newVy; } return seamQualities.Count > 0 ? seamQualities.Average() : 0; } finally { foreach (var s in smallFrames) s.Dispose(); warpMatrix?.Dispose(); } } private static void PasteMini(Mat canvas, int canvasSize, Mat frame, int canvasX, int canvasY) { var srcX = Math.Max(0, -canvasX); var srcY = Math.Max(0, -canvasY); var dstX = Math.Max(0, canvasX); var dstY = Math.Max(0, canvasY); var w = Math.Min(frame.Width - srcX, canvasSize - dstX); var h = Math.Min(frame.Height - srcY, canvasSize - dstY); if (w <= 0 || h <= 0) return; using var srcRoi = new Mat(frame, new Rect(srcX, srcY, w, h)); using var dstRoi = new Mat(canvas, new Rect(dstX, dstY, w, h)); srcRoi.CopyTo(dstRoi); } public void Dispose() { foreach (var f in _frames) f.Dispose(); _frames.Clear(); _lastKept?.Dispose(); } }