From 802f1030d56f2bc0b6a2966fc91d4cee02d6c715 Mon Sep 17 00:00:00 2001 From: Boki Date: Fri, 13 Feb 2026 11:23:30 -0500 Subject: [PATCH] work on navigation --- .../DesktopDuplication.cs | 148 ++++++++++++ src/Poe2Trade.Navigation/GdiCapture.cs | 22 ++ src/Poe2Trade.Navigation/IScreenCapture.cs | 9 + src/Poe2Trade.Navigation/MinimapCapture.cs | 97 ++++++-- .../NavigationExecutor.cs | 16 ++ src/Poe2Trade.Navigation/NavigationTypes.cs | 13 +- .../Poe2Trade.Navigation.csproj | 2 + src/Poe2Trade.Navigation/WgcCapture.cs | 223 ++++++++++++++++++ src/Poe2Trade.Ui/App.axaml.cs | 1 + src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs | 15 ++ .../ViewModels/MainWindowViewModel.cs | 97 +++++--- src/Poe2Trade.Ui/Views/MainWindow.axaml | 3 +- 12 files changed, 582 insertions(+), 64 deletions(-) create mode 100644 src/Poe2Trade.Navigation/DesktopDuplication.cs create mode 100644 src/Poe2Trade.Navigation/GdiCapture.cs create mode 100644 src/Poe2Trade.Navigation/IScreenCapture.cs create mode 100644 src/Poe2Trade.Navigation/WgcCapture.cs diff --git a/src/Poe2Trade.Navigation/DesktopDuplication.cs b/src/Poe2Trade.Navigation/DesktopDuplication.cs new file mode 100644 index 0000000..b9aac3a --- /dev/null +++ b/src/Poe2Trade.Navigation/DesktopDuplication.cs @@ -0,0 +1,148 @@ +using System.Runtime.InteropServices; +using OpenCvSharp; +using Serilog; +using SharpGen.Runtime; +using Vortice.Direct3D; +using Vortice.Direct3D11; +using Vortice.DXGI; +using Region = Poe2Trade.Core.Region; + +namespace Poe2Trade.Navigation; + +public sealed class DesktopDuplication : IScreenCapture +{ + private readonly ID3D11Device _device; + private readonly ID3D11DeviceContext _context; + private IDXGIOutputDuplication? _duplication; + private ID3D11Texture2D? _staging; + private int _stagingW, _stagingH; + private bool _needsRecreate; + + public DesktopDuplication() + { + D3D11.D3D11CreateDevice( + null, + DriverType.Hardware, + DeviceCreationFlags.BgraSupport, + [FeatureLevel.Level_11_0], + out _device!, + out _context!).CheckError(); + + CreateDuplication(); + } + + private void CreateDuplication() + { + using var dxgiDevice = _device.QueryInterface(); + using var adapter = dxgiDevice.GetAdapter(); + adapter.EnumOutputs(0, out var output); + using var _ = output; + using var output1 = output.QueryInterface(); + _duplication = output1.DuplicateOutput(_device); + _needsRecreate = false; + Log.Debug("DXGI Desktop Duplication created"); + } + + public unsafe Mat? CaptureRegion(Region region) + { + if (_duplication == null) return null; + + if (_needsRecreate) + { + try + { + _duplication.Dispose(); + CreateDuplication(); + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to recreate DXGI duplication"); + return null; + } + } + + IDXGIResource? resource = null; + try + { + _duplication.AcquireNextFrame(100, out _, out resource); + } + catch (SharpGenException ex) + { + if (ex.ResultCode == Vortice.DXGI.ResultCode.AccessLost) + _needsRecreate = true; + // WaitTimeout is normal when screen hasn't changed + return null; + } + + try + { + using var srcTexture = resource!.QueryInterface(); + EnsureStaging(region.Width, region.Height); + + // Copy only the region we need from the desktop texture + _context.CopySubresourceRegion( + _staging!, 0, 0, 0, 0, + srcTexture, 0, + new Vortice.Mathematics.Box(region.X, region.Y, 0, + region.X + region.Width, region.Y + region.Height, 1)); + + var mapped = _context.Map(_staging!, 0, MapMode.Read); + try + { + var mat = new Mat(region.Height, region.Width, MatType.CV_8UC4); + var rowBytes = region.Width * 4; + + for (var row = 0; row < region.Height; row++) + { + Buffer.MemoryCopy( + (void*)(mapped.DataPointer + row * mapped.RowPitch), + (void*)mat.Ptr(row), + rowBytes, rowBytes); + } + + // BGRA → BGR + var bgr = new Mat(); + Cv2.CvtColor(mat, bgr, ColorConversionCodes.BGRA2BGR); + mat.Dispose(); + return bgr; + } + finally + { + _context.Unmap(_staging!, 0); + } + } + finally + { + resource?.Dispose(); + _duplication!.ReleaseFrame(); + } + } + + private void EnsureStaging(int w, int h) + { + if (_staging != null && _stagingW == w && _stagingH == h) return; + _staging?.Dispose(); + + _staging = _device.CreateTexture2D(new Texture2DDescription + { + Width = (uint)w, + Height = (uint)h, + MipLevels = 1, + ArraySize = 1, + Format = Format.B8G8R8A8_UNorm, + SampleDescription = new SampleDescription(1, 0), + Usage = ResourceUsage.Staging, + CPUAccessFlags = CpuAccessFlags.Read, + }); + _stagingW = w; + _stagingH = h; + } + + public void Dispose() + { + _staging?.Dispose(); + _duplication?.Dispose(); + _context?.Dispose(); + _device?.Dispose(); + } +} diff --git a/src/Poe2Trade.Navigation/GdiCapture.cs b/src/Poe2Trade.Navigation/GdiCapture.cs new file mode 100644 index 0000000..44799f8 --- /dev/null +++ b/src/Poe2Trade.Navigation/GdiCapture.cs @@ -0,0 +1,22 @@ +using System.Drawing; +using System.Drawing.Imaging; +using OpenCvSharp; +using OpenCvSharp.Extensions; +using Region = Poe2Trade.Core.Region; + +namespace Poe2Trade.Navigation; + +public sealed class GdiCapture : IScreenCapture +{ + public Mat? CaptureRegion(Region region) + { + using var bitmap = new Bitmap(region.Width, region.Height, PixelFormat.Format32bppArgb); + using var g = Graphics.FromImage(bitmap); + g.CopyFromScreen(region.X, region.Y, 0, 0, + new System.Drawing.Size(region.Width, region.Height), + CopyPixelOperation.SourceCopy); + return BitmapConverter.ToMat(bitmap); + } + + public void Dispose() { } +} diff --git a/src/Poe2Trade.Navigation/IScreenCapture.cs b/src/Poe2Trade.Navigation/IScreenCapture.cs new file mode 100644 index 0000000..faff0ce --- /dev/null +++ b/src/Poe2Trade.Navigation/IScreenCapture.cs @@ -0,0 +1,9 @@ +using OpenCvSharp; +using Region = Poe2Trade.Core.Region; + +namespace Poe2Trade.Navigation; + +public interface IScreenCapture : IDisposable +{ + Mat? CaptureRegion(Region region); +} diff --git a/src/Poe2Trade.Navigation/MinimapCapture.cs b/src/Poe2Trade.Navigation/MinimapCapture.cs index b411318..9647109 100644 --- a/src/Poe2Trade.Navigation/MinimapCapture.cs +++ b/src/Poe2Trade.Navigation/MinimapCapture.cs @@ -1,8 +1,4 @@ -using System.Drawing; -using System.Drawing.Imaging; -using System.Runtime.InteropServices; using OpenCvSharp; -using OpenCvSharp.Extensions; using Serilog; using Region = Poe2Trade.Core.Region; using Point = OpenCvSharp.Point; @@ -13,17 +9,45 @@ namespace Poe2Trade.Navigation; public class MinimapCapture : IDisposable { private readonly MinimapConfig _config; + private readonly IScreenCapture _backend; private Mat? _circularMask; - [DllImport("user32.dll")] - private static extern int GetSystemMetrics(int nIndex); - public MinimapCapture(MinimapConfig config) { _config = config; + _backend = CreateBackend(); BuildCircularMask(); } + private static IScreenCapture CreateBackend() + { + // WGC primary → DXGI fallback → GDI last resort + try + { + var wgc = new WgcCapture(); + Log.Information("Screen capture: WGC (Windows Graphics Capture)"); + return wgc; + } + catch (Exception ex) + { + Log.Warning(ex, "WGC unavailable, trying DXGI Desktop Duplication"); + } + + try + { + var dxgi = new DesktopDuplication(); + Log.Information("Screen capture: DXGI Desktop Duplication"); + return dxgi; + } + catch (Exception ex) + { + Log.Warning(ex, "DXGI unavailable, falling back to GDI"); + } + + Log.Information("Screen capture: GDI (CopyFromScreen)"); + return new GdiCapture(); + } + private void BuildCircularMask() { var size = _config.CaptureSize; @@ -35,10 +59,9 @@ public class MinimapCapture : IDisposable public MinimapFrame? CaptureFrame() { var region = _config.CaptureRegion; - using var bitmap = CaptureScreen(region); - using var bgr = BitmapConverter.ToMat(bitmap); + using var bgr = _backend.CaptureRegion(region); - if (bgr.Empty()) + if (bgr == null || bgr.Empty()) return null; // Apply circular mask to ignore area outside fog-of-war circle @@ -64,6 +87,10 @@ public class MinimapCapture : IDisposable Cv2.MorphologyEx(darkMask, wallMask, MorphTypes.Close, wallKernel); // Only within circular mask Cv2.BitwiseAnd(wallMask, _circularMask!, wallMask); + // Don't count explored pixels as wall + using var notExplored = new Mat(); + Cv2.BitwiseNot(exploredMask, notExplored); + Cv2.BitwiseAnd(wallMask, notExplored, wallMask); // Build classified mat: Unknown=0, Explored=1, Wall=2 var classified = new Mat(_config.CaptureSize, _config.CaptureSize, MatType.CV_8UC1, Scalar.Black); @@ -91,6 +118,45 @@ public class MinimapCapture : IDisposable ); } + /// + /// Save debug images: raw capture, HSV mask, classified result. + /// Call once to diagnose color ranges. + /// + public void SaveDebugCapture(string dir = "debug-minimap") + { + Directory.CreateDirectory(dir); + var region = _config.CaptureRegion; + using var bgr = _backend.CaptureRegion(region); + if (bgr == null || bgr.Empty()) return; + + using var masked = new Mat(); + Cv2.BitwiseAnd(bgr, bgr, masked, _circularMask!); + + using var hsv = new Mat(); + Cv2.CvtColor(masked, hsv, ColorConversionCodes.BGR2HSV); + + using var exploredMask = new Mat(); + Cv2.InRange(hsv, _config.ExploredLoHSV, _config.ExploredHiHSV, exploredMask); + + using var playerMask = new Mat(); + Cv2.InRange(hsv, _config.PlayerLoHSV, _config.PlayerHiHSV, playerMask); + + // Save raw, masked, explored filter, player filter + Cv2.ImWrite(Path.Combine(dir, "1-raw.png"), bgr); + Cv2.ImWrite(Path.Combine(dir, "2-masked.png"), masked); + Cv2.ImWrite(Path.Combine(dir, "3-explored.png"), exploredMask); + Cv2.ImWrite(Path.Combine(dir, "4-player.png"), playerMask); + + // Save HSV channels separately for tuning + var channels = Cv2.Split(hsv); + Cv2.ImWrite(Path.Combine(dir, "5-hue.png"), channels[0]); + Cv2.ImWrite(Path.Combine(dir, "6-sat.png"), channels[1]); + Cv2.ImWrite(Path.Combine(dir, "7-val.png"), channels[2]); + foreach (var c in channels) c.Dispose(); + + Log.Information("Debug minimap images saved to {Dir}", Path.GetFullPath(dir)); + } + private Point2d FindCentroid(Mat mask) { var moments = Cv2.Moments(mask, true); @@ -102,18 +168,9 @@ public class MinimapCapture : IDisposable return new Point2d(cx, cy); } - private static Bitmap CaptureScreen(Region region) - { - var bitmap = new Bitmap(region.Width, region.Height, PixelFormat.Format32bppArgb); - using var g = Graphics.FromImage(bitmap); - g.CopyFromScreen(region.X, region.Y, 0, 0, - new System.Drawing.Size(region.Width, region.Height), - CopyPixelOperation.SourceCopy); - return bitmap; - } - public void Dispose() { _circularMask?.Dispose(); + _backend.Dispose(); } } diff --git a/src/Poe2Trade.Navigation/NavigationExecutor.cs b/src/Poe2Trade.Navigation/NavigationExecutor.cs index b0997dd..0e6aabf 100644 --- a/src/Poe2Trade.Navigation/NavigationExecutor.cs +++ b/src/Poe2Trade.Navigation/NavigationExecutor.cs @@ -147,6 +147,22 @@ public class NavigationExecutor : IDisposable public byte[] GetMapSnapshot() => _worldMap.GetMapSnapshot(); public byte[] GetViewportSnapshot(int viewSize = 400) => _worldMap.GetViewportSnapshot(_tracker.Position, viewSize); + /// + /// Capture one frame, track position, stitch into world map. + /// Returns viewport PNG bytes, or null on failure. + /// + public byte[]? ProcessFrame() + { + using var frame = _capture.CaptureFrame(); + if (frame == null) return null; + + var pos = _tracker.UpdatePosition(frame.GrayMat); + _worldMap.StitchFrame(frame.ClassifiedMat, pos); + return _worldMap.GetViewportSnapshot(pos); + } + + public void SaveDebugCapture() => _capture.SaveDebugCapture(); + public void Dispose() { _capture.Dispose(); diff --git a/src/Poe2Trade.Navigation/NavigationTypes.cs b/src/Poe2Trade.Navigation/NavigationTypes.cs index 30075ad..8591891 100644 --- a/src/Poe2Trade.Navigation/NavigationTypes.cs +++ b/src/Poe2Trade.Navigation/NavigationTypes.cs @@ -55,16 +55,17 @@ public class MinimapConfig // Fog-of-war circle radius within the captured frame public int FogRadius { get; set; } = 120; - // HSV range for explored areas (blue/teal) - public Scalar ExploredLoHSV { get; set; } = new(85, 30, 30); - public Scalar ExploredHiHSV { get; set; } = new(145, 255, 255); + // HSV range for explored areas (purple/violet minimap lines) + // OpenCV H: 0-180, purple ≈ 120-170 + public Scalar ExploredLoHSV { get; set; } = new(120, 40, 40); + public Scalar ExploredHiHSV { get; set; } = new(170, 255, 255); // HSV range for player marker (orange X) - public Scalar PlayerLoHSV { get; set; } = new(5, 100, 100); + public Scalar PlayerLoHSV { get; set; } = new(5, 80, 80); public Scalar PlayerHiHSV { get; set; } = new(25, 255, 255); - // HSV range for walls (dark pixels) - public int WallMaxValue { get; set; } = 40; + // HSV range for walls (dark pixels) — set very low to avoid game-world noise + public int WallMaxValue { get; set; } = 20; // Movement public int ClickRadius { get; set; } = 100; diff --git a/src/Poe2Trade.Navigation/Poe2Trade.Navigation.csproj b/src/Poe2Trade.Navigation/Poe2Trade.Navigation.csproj index a14735f..a85875e 100644 --- a/src/Poe2Trade.Navigation/Poe2Trade.Navigation.csproj +++ b/src/Poe2Trade.Navigation/Poe2Trade.Navigation.csproj @@ -10,6 +10,8 @@ + + diff --git a/src/Poe2Trade.Navigation/WgcCapture.cs b/src/Poe2Trade.Navigation/WgcCapture.cs new file mode 100644 index 0000000..e8bc9e2 --- /dev/null +++ b/src/Poe2Trade.Navigation/WgcCapture.cs @@ -0,0 +1,223 @@ +using System.Runtime.InteropServices; +using OpenCvSharp; +using Serilog; +using Vortice.Direct3D; +using Vortice.Direct3D11; +using Vortice.DXGI; +using Windows.Graphics.Capture; +using Windows.Graphics.DirectX; +using Windows.Graphics.DirectX.Direct3D11; +using Region = Poe2Trade.Core.Region; + +namespace Poe2Trade.Navigation; + +public sealed class WgcCapture : IScreenCapture +{ + // IGraphicsCaptureItemInterop extends IInspectable (not IUnknown), + // so we need 3 padding methods for the IInspectable vtable slots. + [ComImport, Guid("3628E81B-3CAC-4C60-B7F4-23CE0E0C3356")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface IGraphicsCaptureItemInterop + { + void _pakGetIids(); + void _pakGetRuntimeClassName(); + void _pakGetTrustLevel(); + IntPtr CreateForWindow([In] IntPtr window, [In] ref Guid iid); + IntPtr CreateForMonitor([In] IntPtr monitor, [In] ref Guid iid); + } + + // IDirect3DDxgiInterfaceAccess extends IUnknown directly — no padding needed. + [ComImport, Guid("A9B3D012-3DF2-4EE3-B8D1-8695F457D3C1")] + [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] + private interface IDirect3DDxgiInterfaceAccess + { + IntPtr GetInterface([In] ref Guid iid); + } + + [DllImport("d3d11.dll", EntryPoint = "CreateDirect3D11DeviceFromDXGIDevice")] + private static extern int CreateDirect3D11DeviceFromDXGIDevice(IntPtr dxgiDevice, out IntPtr graphicsDevice); + + [DllImport("user32.dll")] + private static extern IntPtr MonitorFromPoint(POINT pt, uint dwFlags); + + [DllImport("combase.dll")] + private static extern int RoGetActivationFactory(IntPtr classId, ref Guid iid, out IntPtr factory); + + [DllImport("combase.dll")] + private static extern int WindowsCreateString( + [MarshalAs(UnmanagedType.LPWStr)] string src, int len, out IntPtr hstr); + + [DllImport("combase.dll")] + private static extern int WindowsDeleteString(IntPtr hstr); + + [StructLayout(LayoutKind.Sequential)] + private struct POINT { public int X, Y; } + + private const uint MONITOR_DEFAULTTOPRIMARY = 1; + + private readonly ID3D11Device _device; + private readonly ID3D11DeviceContext _context; + private readonly Direct3D11CaptureFramePool _framePool; + private readonly GraphicsCaptureSession _session; + private ID3D11Texture2D? _staging; + private int _stagingW, _stagingH; + + public WgcCapture() + { + D3D11.D3D11CreateDevice( + null, DriverType.Hardware, DeviceCreationFlags.BgraSupport, + [FeatureLevel.Level_11_0], + out _device!, out _context!).CheckError(); + + // Wrap D3D11 device as WinRT IDirect3DDevice + using var dxgiDev = _device.QueryInterface(); + Marshal.ThrowExceptionForHR( + CreateDirect3D11DeviceFromDXGIDevice(dxgiDev.NativePointer, out var inspectable)); + var winrtDevice = (IDirect3DDevice)Marshal.GetObjectForIUnknown(inspectable); + Marshal.Release(inspectable); + + // Get primary monitor + var hMon = MonitorFromPoint(default, MONITOR_DEFAULTTOPRIMARY); + + // Create GraphicsCaptureItem from HMONITOR via interop factory + var item = CreateCaptureItemForMonitor(hMon); + + // Free-threaded pool: no STA required, supports polling via TryGetNextFrame + _framePool = Direct3D11CaptureFramePool.CreateFreeThreaded( + winrtDevice, DirectXPixelFormat.B8G8R8A8UIntNormalized, + 1, item.Size); + + _session = _framePool.CreateCaptureSession(item); + _session.StartCapture(); + Log.Debug("WGC capture started ({W}x{H})", item.Size.Width, item.Size.Height); + } + + private static GraphicsCaptureItem CreateCaptureItemForMonitor(IntPtr hMonitor) + { + const string className = "Windows.Graphics.Capture.GraphicsCaptureItem"; + var interopIid = new Guid("3628E81B-3CAC-4C60-B7F4-23CE0E0C3356"); + + WindowsCreateString(className, className.Length, out var hString); + try + { + Marshal.ThrowExceptionForHR( + RoGetActivationFactory(hString, ref interopIid, out var factoryPtr)); + try + { + var interop = (IGraphicsCaptureItemInterop)Marshal.GetObjectForIUnknown(factoryPtr); + // IGraphicsCaptureItem GUID + var itemIid = new Guid("79C3F95B-31F7-4EC2-A464-632EF5D30760"); + var itemPtr = interop.CreateForMonitor(hMonitor, ref itemIid); + var item = (GraphicsCaptureItem)Marshal.GetObjectForIUnknown(itemPtr); + Marshal.Release(itemPtr); + return item; + } + finally + { + Marshal.Release(factoryPtr); + } + } + finally + { + WindowsDeleteString(hString); + } + } + + public unsafe Mat? CaptureRegion(Region region) + { + using var frame = _framePool.TryGetNextFrame(); + if (frame == null) return null; + + using var srcTexture = GetTextureFromSurface(frame.Surface); + if (srcTexture == null) return null; + + EnsureStaging(region.Width, region.Height); + + _context.CopySubresourceRegion( + _staging!, 0, 0, 0, 0, + srcTexture, 0, + new Vortice.Mathematics.Box(region.X, region.Y, 0, + region.X + region.Width, region.Y + region.Height, 1)); + + var mapped = _context.Map(_staging!, 0, MapMode.Read); + try + { + var mat = new Mat(region.Height, region.Width, MatType.CV_8UC4); + var rowBytes = region.Width * 4; + + for (var row = 0; row < region.Height; row++) + { + Buffer.MemoryCopy( + (void*)(mapped.DataPointer + row * mapped.RowPitch), + (void*)mat.Ptr(row), + rowBytes, rowBytes); + } + + var bgr = new Mat(); + Cv2.CvtColor(mat, bgr, ColorConversionCodes.BGRA2BGR); + mat.Dispose(); + return bgr; + } + finally + { + _context.Unmap(_staging!, 0); + } + } + + private ID3D11Texture2D? GetTextureFromSurface(IDirect3DSurface surface) + { + var unknown = Marshal.GetIUnknownForObject(surface); + try + { + var accessIid = new Guid("A9B3D012-3DF2-4EE3-B8D1-8695F457D3C1"); + var hr = Marshal.QueryInterface(unknown, ref accessIid, out var accessPtr); + if (hr != 0) return null; + + try + { + var access = (IDirect3DDxgiInterfaceAccess)Marshal.GetObjectForIUnknown(accessPtr); + // ID3D11Texture2D GUID + var texIid = new Guid("6f15aaf2-d208-4e89-9ab4-489535d34f9c"); + var texPtr = access.GetInterface(ref texIid); + return new ID3D11Texture2D(texPtr); + } + finally + { + Marshal.Release(accessPtr); + } + } + finally + { + Marshal.Release(unknown); + } + } + + private void EnsureStaging(int w, int h) + { + if (_staging != null && _stagingW == w && _stagingH == h) return; + _staging?.Dispose(); + + _staging = _device.CreateTexture2D(new Texture2DDescription + { + Width = (uint)w, + Height = (uint)h, + MipLevels = 1, + ArraySize = 1, + Format = Format.B8G8R8A8_UNorm, + SampleDescription = new SampleDescription(1, 0), + Usage = ResourceUsage.Staging, + CPUAccessFlags = CpuAccessFlags.Read, + }); + _stagingW = w; + _stagingH = h; + } + + public void Dispose() + { + _session?.Dispose(); + _framePool?.Dispose(); + _staging?.Dispose(); + _context?.Dispose(); + _device?.Dispose(); + } +} diff --git a/src/Poe2Trade.Ui/App.axaml.cs b/src/Poe2Trade.Ui/App.axaml.cs index 52aef57..c49b743 100644 --- a/src/Poe2Trade.Ui/App.axaml.cs +++ b/src/Poe2Trade.Ui/App.axaml.cs @@ -67,6 +67,7 @@ public partial class App : Application desktop.ShutdownRequested += async (_, _) => { + mainVm.Shutdown(); await bot.DisposeAsync(); }; } diff --git a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs index f9fa122..6eca8e1 100644 --- a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs @@ -160,6 +160,21 @@ public partial class DebugViewModel : ObservableObject } } + [RelayCommand] + private void SaveMinimapDebug() + { + try + { + _bot.Navigation.SaveDebugCapture(); + DebugResult = $"Minimap debug images saved to {Path.GetFullPath("debug-minimap")}"; + } + catch (Exception ex) + { + DebugResult = $"Failed: {ex.Message}"; + Log.Error(ex, "Minimap debug save failed"); + } + } + [RelayCommand] private async Task ClickAnge() { diff --git a/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs b/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs index 116d020..997fc6e 100644 --- a/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs @@ -1,5 +1,6 @@ using System.Collections.ObjectModel; using System.IO; +using System.Runtime.InteropServices; using Avalonia.Media.Imaging; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; @@ -29,6 +30,11 @@ public partial class CellState : ObservableObject public partial class MainWindowViewModel : ObservableObject { private readonly BotOrchestrator _bot; + private readonly CancellationTokenSource _cts = new(); + + [DllImport("user32.dll")] + private static extern short GetAsyncKeyState(int vKey); + private const int VK_F12 = 0x7B; [ObservableProperty] private string _state = "Idle"; @@ -45,7 +51,6 @@ public partial class MainWindowViewModel : ObservableObject [ObservableProperty] private Bitmap? _inventoryImage; [ObservableProperty] private Bitmap? _minimapImage; [ObservableProperty] private string _navigationStateText = ""; - private long _lastMinimapUpdate; [ObservableProperty] private string _newUrl = ""; [ObservableProperty] private string _newLinkName = ""; @@ -80,7 +85,6 @@ public partial class MainWindowViewModel : ObservableObject ActiveLinksCount = status.Links.Count(l => l.Active); OnPropertyChanged(nameof(Links)); UpdateInventoryGrid(); - UpdateMinimapImage(); }); }; @@ -97,6 +101,9 @@ public partial class MainWindowViewModel : ObservableObject if (Logs.Count > 500) Logs.RemoveAt(0); }); }; + + // Background loop: minimap capture + F12 hotkey polling + _ = RunBackgroundLoop(_cts.Token); } public string PauseButtonText => IsPaused ? "Resume" : "Pause"; @@ -167,6 +174,57 @@ public partial class MainWindowViewModel : ObservableObject if (link != null) _bot.ToggleLink(id, !link.Active); } + private async Task RunBackgroundLoop(CancellationToken ct) + { + var f12WasDown = false; + + while (!ct.IsCancellationRequested) + { + try + { + // F12 hotkey — edge-detect (trigger once per press) + var f12Down = (GetAsyncKeyState(VK_F12) & 0x8000) != 0; + if (f12Down && !f12WasDown) + { + Log.Information("F12 pressed — emergency stop"); + await _bot.Navigation.Stop(); + _bot.Pause(); + Avalonia.Threading.Dispatcher.UIThread.Post(() => + { + IsPaused = true; + State = "Stopped (F12)"; + }); + } + f12WasDown = f12Down; + + // Minimap capture + display + var bytes = _bot.Navigation.ProcessFrame(); + if (bytes != null) + { + var bmp = new Bitmap(new MemoryStream(bytes)); + Avalonia.Threading.Dispatcher.UIThread.Post(() => + { + var old = MinimapImage; + MinimapImage = bmp; + NavigationStateText = _bot.Navigation.State == NavigationState.Idle + ? "" : _bot.Navigation.State.ToString(); + old?.Dispose(); + }); + } + + await Task.Delay(100, ct); + } + catch (OperationCanceledException) { break; } + catch (Exception ex) + { + Log.Debug(ex, "Background loop error"); + await Task.Delay(500, ct); + } + } + } + + public void Shutdown() => _cts.Cancel(); + private void UpdateInventoryGrid() { if (!_bot.IsReady) return; @@ -213,39 +271,4 @@ public partial class MainWindowViewModel : ObservableObject OnPropertyChanged(nameof(InventoryFreeCells)); } - - private void UpdateMinimapImage() - { - var nav = _bot.Navigation; - var navState = nav.State; - NavigationStateText = navState == NavigationState.Idle ? "" : navState.ToString(); - - if (navState == NavigationState.Idle) - { - if (MinimapImage != null) - { - var old = MinimapImage; - MinimapImage = null; - old.Dispose(); - } - return; - } - - // Throttle: update at most once per second - var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - if (now - _lastMinimapUpdate < 1000) return; - _lastMinimapUpdate = now; - - try - { - var bytes = nav.GetViewportSnapshot(); - var old = MinimapImage; - MinimapImage = new Bitmap(new MemoryStream(bytes)); - old?.Dispose(); - } - catch (Exception ex) - { - Log.Debug(ex, "Failed to update minimap image"); - } - } } diff --git a/src/Poe2Trade.Ui/Views/MainWindow.axaml b/src/Poe2Trade.Ui/Views/MainWindow.axaml index 109af8f..3ce19cf 100644 --- a/src/Poe2Trade.Ui/Views/MainWindow.axaml +++ b/src/Poe2Trade.Ui/Views/MainWindow.axaml @@ -115,7 +115,7 @@ + FontSize="11" Foreground="#58a6ff" Margin="8,0,0,0" />