diff --git a/.gitignore b/.gitignore index 4f919d8..7290581 100644 --- a/.gitignore +++ b/.gitignore @@ -18,8 +18,16 @@ config.json browser-data/ *.log debug-screenshots/ +debug/ items/ +# YOLO / ML +runs/ +training-data/ +tools/python-detect/models/ +*.pt +*.engine + # IDE / tools .claude/ nul diff --git a/assets/black-cathedral.png b/assets/black-cathedral.png new file mode 100644 index 0000000..3138314 Binary files /dev/null and b/assets/black-cathedral.png differ diff --git a/assets/invitation.png b/assets/invitation.png new file mode 100644 index 0000000..2f3f672 Binary files /dev/null and b/assets/invitation.png differ diff --git a/assets/well-of-souls.png b/assets/well-of-souls.png new file mode 100644 index 0000000..51425fe Binary files /dev/null and b/assets/well-of-souls.png differ diff --git a/src/Poe2Trade.Bot/BossRunExecutor.cs b/src/Poe2Trade.Bot/BossRunExecutor.cs new file mode 100644 index 0000000..8132ec9 --- /dev/null +++ b/src/Poe2Trade.Bot/BossRunExecutor.cs @@ -0,0 +1,529 @@ +using System.Diagnostics; +using Poe2Trade.Core; +using Poe2Trade.Game; +using Poe2Trade.GameLog; +using Poe2Trade.Inventory; +using Poe2Trade.Screen; +using Serilog; + +namespace Poe2Trade.Bot; + +public class BossRunExecutor +{ + private static readonly string WellOfSoulsTemplate = Path.Combine("assets", "well-of-souls.png"); + private static readonly string BlackCathedralTemplate = Path.Combine("assets", "black-cathedral.png"); + private static readonly string InvitationTemplate = Path.Combine("assets", "invitation.png"); + + private BossRunState _state = BossRunState.Idle; + private bool _stopped; + private readonly IGameController _game; + private readonly IScreenReader _screen; + private readonly IInventoryManager _inventory; + private readonly IClientLogWatcher _logWatcher; + private readonly SavedSettings _config; + private readonly BossDetector _bossDetector; + + public event Action? StateChanged; + + public BossRunExecutor(IGameController game, IScreenReader screen, + IInventoryManager inventory, IClientLogWatcher logWatcher, SavedSettings config, + BossDetector bossDetector) + { + _game = game; + _screen = screen; + _inventory = inventory; + _logWatcher = logWatcher; + _config = config; + _bossDetector = bossDetector; + } + + public BossRunState State => _state; + + private void SetState(BossRunState s) + { + _state = s; + StateChanged?.Invoke(s); + } + + public void Stop() + { + _stopped = true; + Log.Information("Boss run executor stop requested"); + } + + public async Task RunBossLoop() + { + _stopped = false; + _bossDetector.SetBoss("kulemak"); + Log.Information("Starting boss run loop ({Count} invitations)", _config.Kulemak.InvitationCount); + + if (!await Prepare()) + { + SetState(BossRunState.Failed); + await RecoverToHideout(); + SetState(BossRunState.Idle); + return; + } + + var completed = 0; + for (var i = 0; i < _config.Kulemak.InvitationCount; i++) + { + if (_stopped) break; + + Log.Information("=== Boss run {N}/{Total} ===", i + 1, _config.Kulemak.InvitationCount); + + if (!await TravelToZone()) + { + Log.Error("Failed to travel to zone"); + await RecoverToHideout(); + break; + } + if (_stopped) break; + + var entrance = await WalkToEntrance(); + if (entrance == null) + { + Log.Error("Failed to find Black Cathedral entrance"); + await RecoverToHideout(); + break; + } + if (_stopped) break; + + if (!await UseInvitation(entrance.X, entrance.Y)) + { + Log.Error("Failed to use invitation"); + await RecoverToHideout(); + break; + } + if (_stopped) break; + + await Fight(); + if (_stopped) break; + + await Loot(); + if (_stopped) break; + + if (!await ReturnHome()) + { + Log.Error("Failed to return home"); + await RecoverToHideout(); + break; + } + if (_stopped) break; + + await StoreLoot(); + completed++; + + if (_stopped) break; + } + + Log.Information("Boss run loop finished: {Completed}/{Total} runs completed", completed, _config.Kulemak.InvitationCount); + SetState(BossRunState.Complete); + await Helpers.Sleep(1000); + SetState(BossRunState.Idle); + } + + private async Task Prepare() + { + SetState(BossRunState.Preparing); + Log.Information("Preparing: depositing inventory and grabbing invitations"); + + await _game.FocusGame(); + await Helpers.Sleep(Delays.PostFocus); + + // Open stash + var stashPos = await _inventory.FindAndClickNameplate("Stash"); + if (stashPos == null) + { + Log.Error("Could not find Stash nameplate"); + return false; + } + await Helpers.Sleep(Delays.PostStashOpen); + + // Click loot tab and deposit all inventory items + var (lootTab, lootFolder) = ResolveTabPath(_config.Kulemak.LootTabPath); + if (lootTab != null) + { + await _inventory.ClickStashTab(lootTab, lootFolder); + + // Deposit all inventory items via ctrl+click + var scanResult = await _screen.Grid.Scan("inventory"); + if (scanResult.Occupied.Count > 0) + { + Log.Information("Depositing {Count} inventory items to loot tab", scanResult.Occupied.Count); + await _game.KeyDown(InputSender.VK.SHIFT); + await _game.HoldCtrl(); + foreach (var cell in scanResult.Occupied) + { + var center = _screen.Grid.GetCellCenter(GridLayouts.Inventory, cell.Row, cell.Col); + await _game.LeftClickAt(center.X, center.Y); + await Helpers.Sleep(Delays.ClickInterval); + } + await _game.ReleaseCtrl(); + await _game.KeyUp(InputSender.VK.SHIFT); + await Helpers.Sleep(Delays.PostEscape); + } + } + else + { + Log.Warning("Loot tab path not configured or not found, skipping deposit"); + } + + // Click invitation tab and grab invitations + var (invTab, invFolder) = ResolveTabPath(_config.Kulemak.InvitationTabPath); + if (invTab != null) + { + await _inventory.ClickStashTab(invTab, invFolder); + + // Determine layout name based on tab config + var layoutName = (invTab.GridCols == 24, invFolder != null) switch + { + (true, true) => "stash24_folder", + (true, false) => "stash24", + (false, true) => "stash12_folder", + (false, false) => "stash12", + }; + + await _inventory.GrabItemsFromStash(layoutName, _config.Kulemak.InvitationCount, InvitationTemplate); + } + else + { + Log.Warning("Invitation tab path not configured or not found, skipping grab"); + } + + // Close stash + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + + Log.Information("Preparation complete"); + return true; + } + + private async Task TravelToZone() + { + SetState(BossRunState.TravelingToZone); + Log.Information("Traveling to Well of Souls via waypoint"); + + await _game.FocusGame(); + await Helpers.Sleep(Delays.PostFocus); + + // Find and click Waypoint + var wpPos = await _inventory.FindAndClickNameplate("Waypoint"); + if (wpPos == null) + { + Log.Error("Could not find Waypoint nameplate"); + return false; + } + await Helpers.Sleep(1000); + + // Template match well-of-souls.png and click + var match = await _screen.TemplateMatch(WellOfSoulsTemplate); + if (match == null) + { + Log.Error("Could not find Well of Souls on waypoint map"); + await _game.PressEscape(); + return false; + } + + Log.Information("Found Well of Souls at ({X},{Y}), clicking", match.X, match.Y); + await _game.LeftClickAt(match.X, match.Y); + + // Wait for area transition + var arrived = await _inventory.WaitForAreaTransition(_config.TravelTimeoutMs); + if (!arrived) + { + Log.Error("Timed out waiting for Well of Souls transition"); + return false; + } + + await Helpers.Sleep(Delays.PostTravel); + Log.Information("Arrived at Well of Souls"); + return true; + } + + private async Task WalkToEntrance() + { + SetState(BossRunState.WalkingToEntrance); + Log.Information("Walking to Black Cathedral entrance (W+D)"); + + return await WalkAndMatch(BlackCathedralTemplate, InputSender.VK.W, InputSender.VK.D, 15000); + } + + private async Task UseInvitation(int x, int y) + { + SetState(BossRunState.UsingInvitation); + Log.Information("Using invitation at ({X},{Y})", x, y); + + // Hover first so the game registers the target, then use invitation + await _game.MoveMouseTo(x, y); + await Helpers.Sleep(500); + await _game.CtrlLeftClickAt(x, y); + await Helpers.Sleep(1000); + + // Find "NEW" text — pick the leftmost instance + var ocr = await _screen.Ocr(); + var newWords = ocr.Lines + .SelectMany(l => l.Words) + .Where(w => w.Text.Equals("NEW", StringComparison.OrdinalIgnoreCase) + || w.Text.Equals("New", StringComparison.Ordinal)) + .OrderBy(w => w.X) + .ToList(); + + if (newWords.Count == 0) + { + Log.Error("Could not find 'NEW' text for instance selection"); + return false; + } + + var target = newWords[0]; + var clickX = target.X + target.Width / 2; + var clickY = target.Y + target.Height / 2; + Log.Information("Found {Count} 'NEW' matches, clicking leftmost at ({X},{Y})", newWords.Count, clickX, clickY); + await _game.MoveMouseTo(clickX, clickY); + await Helpers.Sleep(500); + await _game.LeftClickAt(clickX, clickY); + + // Wait for area transition into boss arena + var arrived = await _inventory.WaitForAreaTransition(_config.TravelTimeoutMs); + if (!arrived) + { + Log.Error("Timed out waiting for boss arena transition"); + return false; + } + + await Helpers.Sleep(Delays.PostTravel); + Log.Information("Entered boss arena"); + return true; + } + + private async Task Fight() + { + SetState(BossRunState.Fighting); + Log.Information("[PLACEHOLDER] Fight phase - waiting for manual combat"); + // Placeholder: user handles combat manually for now + await Helpers.Sleep(1000); + } + + private async Task Loot() + { + SetState(BossRunState.Looting); + Log.Information("[PLACEHOLDER] Loot phase - waiting for manual looting"); + // Placeholder: user handles looting manually for now + await Helpers.Sleep(1000); + } + + private async Task ReturnHome() + { + SetState(BossRunState.Returning); + Log.Information("Returning home"); + + await _game.FocusGame(); + await Helpers.Sleep(Delays.PostFocus); + + // Walk away from loot (hold S briefly) + await _game.KeyDown(InputSender.VK.S); + await Helpers.Sleep(1000); + await _game.KeyUp(InputSender.VK.S); + await Helpers.Sleep(300); + + // Press + to open portal + await _game.PressPlus(); + await Helpers.Sleep(1500); + + // Find "The Ardura Caravan" and click it + var caravanPos = await _inventory.FindAndClickNameplate("The Ardura Caravan", maxRetries: 5, retryDelayMs: 1500); + if (caravanPos == null) + { + Log.Error("Could not find 'The Ardura Caravan' portal"); + return false; + } + + // Wait for area transition to caravan + var arrivedCaravan = await _inventory.WaitForAreaTransition(_config.TravelTimeoutMs); + if (!arrivedCaravan) + { + Log.Error("Timed out waiting for caravan transition"); + return false; + } + await Helpers.Sleep(Delays.PostTravel); + + // /hideout to go home + var arrivedHome = await _inventory.WaitForAreaTransition( + _config.TravelTimeoutMs, () => _game.GoToHideout()); + if (!arrivedHome) + { + Log.Error("Timed out going to hideout"); + return false; + } + + await Helpers.Sleep(Delays.PostTravel); + _inventory.SetLocation(true); + Log.Information("Arrived at hideout"); + return true; + } + + private async Task StoreLoot() + { + SetState(BossRunState.StoringLoot); + Log.Information("Storing loot"); + + await _game.FocusGame(); + await Helpers.Sleep(Delays.PostFocus); + + // Open stash + var stashPos = await _inventory.FindAndClickNameplate("Stash"); + if (stashPos == null) + { + Log.Warning("Could not find Stash, skipping loot storage"); + return; + } + await Helpers.Sleep(Delays.PostStashOpen); + + // Click loot tab + var (lootTab, lootFolder) = ResolveTabPath(_config.Kulemak.LootTabPath); + if (lootTab != null) + await _inventory.ClickStashTab(lootTab, lootFolder); + + // Deposit all inventory items + var scanResult = await _screen.Grid.Scan("inventory"); + if (scanResult.Occupied.Count > 0) + { + Log.Information("Depositing {Count} items to loot tab", scanResult.Occupied.Count); + await _game.KeyDown(InputSender.VK.SHIFT); + await _game.HoldCtrl(); + foreach (var cell in scanResult.Occupied) + { + var center = _screen.Grid.GetCellCenter(GridLayouts.Inventory, cell.Row, cell.Col); + await _game.LeftClickAt(center.X, center.Y); + await Helpers.Sleep(Delays.ClickInterval); + } + await _game.ReleaseCtrl(); + await _game.KeyUp(InputSender.VK.SHIFT); + await Helpers.Sleep(Delays.PostEscape); + } + + // Close stash + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + + Log.Information("Loot stored"); + } + + private async Task WalkAndMatch(string templatePath, int vk1, int vk2, + int timeoutMs = 15000, int closeRadius = 350) + { + const int screenCx = 2560 / 2; + const int screenCy = 1440 / 2; + + await _game.KeyDown(vk1); + await _game.KeyDown(vk2); + try + { + var sw = Stopwatch.StartNew(); + bool spotted = false; + while (sw.ElapsedMilliseconds < timeoutMs) + { + if (_stopped) return null; + var match = await _screen.TemplateMatch(templatePath); + if (match == null) + { + await Helpers.Sleep(500); + continue; + } + + var dx = match.X - screenCx; + var dy = match.Y - screenCy; + var dist = Math.Sqrt(dx * dx + dy * dy); + + if (!spotted) + { + Log.Information("Template spotted at ({X},{Y}), dist={Dist:F0}px from center, approaching...", + match.X, match.Y, dist); + spotted = true; + } + + if (dist <= closeRadius) + { + Log.Information("Close enough at ({X},{Y}), dist={Dist:F0}px, stopping", match.X, match.Y, dist); + + // Stop, settle, re-match for accurate position + await _game.KeyUp(vk2); + await _game.KeyUp(vk1); + await Helpers.Sleep(300); + + var fresh = await _screen.TemplateMatch(templatePath); + if (fresh != null) + { + Log.Information("Final position at ({X},{Y})", fresh.X, fresh.Y); + return fresh; + } + Log.Warning("Re-match failed, using last known position"); + return match; + } + + await Helpers.Sleep(200); + } + Log.Error("WalkAndMatch timed out after {Ms}ms (spotted={Spotted})", timeoutMs, spotted); + return null; + } + finally + { + await _game.KeyUp(vk2); + await _game.KeyUp(vk1); + } + } + + private (StashTabInfo? Tab, StashTabInfo? Folder) ResolveTabPath(string tabPath) + { + if (string.IsNullOrEmpty(tabPath) || _config.StashCalibration == null) + return (null, null); + + var parts = tabPath.Split('/'); + if (parts.Length == 1) + { + // Simple tab name + var tab = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0]); + return (tab, null); + } + + if (parts.Length == 2) + { + // Folder/SubTab + var folder = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0] && t.IsFolder); + if (folder == null) return (null, null); + var subTab = folder.SubTabs.FirstOrDefault(t => t.Name == parts[1]); + return (subTab, folder); + } + + return (null, null); + } + + private async Task RecoverToHideout() + { + try + { + Log.Information("Recovering: escaping and going to hideout"); + await _game.FocusGame(); + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + + var arrived = await _inventory.WaitForAreaTransition( + _config.TravelTimeoutMs, () => _game.GoToHideout()); + if (arrived) + { + _inventory.SetLocation(true); + Log.Information("Recovery: arrived at hideout"); + } + else + { + Log.Warning("Recovery: timed out going to hideout"); + } + } + catch (Exception ex) + { + Log.Error(ex, "Recovery failed"); + } + } +} diff --git a/src/Poe2Trade.Bot/BotOrchestrator.cs b/src/Poe2Trade.Bot/BotOrchestrator.cs index 7796f2c..7c60b2c 100644 --- a/src/Poe2Trade.Bot/BotOrchestrator.cs +++ b/src/Poe2Trade.Bot/BotOrchestrator.cs @@ -44,7 +44,9 @@ public class BotOrchestrator : IAsyncDisposable public GameStateDetector GameState { get; } public HudReader HudReader { get; } public EnemyDetector EnemyDetector { get; } + public BossDetector BossDetector { get; } public FrameSaver FrameSaver { get; } + public BossRunExecutor BossRunExecutor { get; } private readonly Dictionary _scrapExecutors = new(); // Events @@ -72,6 +74,7 @@ public class BotOrchestrator : IAsyncDisposable GameState = new GameStateDetector(); HudReader = new HudReader(); EnemyDetector = new EnemyDetector(); + BossDetector = new BossDetector(); FrameSaver = new FrameSaver(); // Register on shared pipeline @@ -79,12 +82,15 @@ public class BotOrchestrator : IAsyncDisposable pipelineService.Pipeline.AddConsumer(GameState); pipelineService.Pipeline.AddConsumer(HudReader); pipelineService.Pipeline.AddConsumer(EnemyDetector); + pipelineService.Pipeline.AddConsumer(BossDetector); pipelineService.Pipeline.AddConsumer(FrameSaver); // Pass shared pipeline to NavigationExecutor Navigation = new NavigationExecutor(game, pipelineService.Pipeline, minimapCapture, enemyDetector: EnemyDetector); + BossRunExecutor = new BossRunExecutor(game, screen, inventory, logWatcher, store.Settings, BossDetector); + logWatcher.AreaEntered += _ => Navigation.Reset(); logWatcher.Start(); // start early so area events fire even before Bot.Start() _paused = store.Settings.Paused; @@ -182,6 +188,11 @@ public class BotOrchestrator : IAsyncDisposable return; } } + if (BossRunExecutor.State != BossRunState.Idle) + { + State = BossRunExecutor.State.ToString(); + return; + } if (Navigation.State != NavigationState.Idle) { State = Navigation.State.ToString(); @@ -264,26 +275,61 @@ public class BotOrchestrator : IAsyncDisposable { LogWatcher.Start(); await Game.FocusGame(); + await Screen.Warmup(); + BossRunExecutor.StateChanged += _ => UpdateExecutorState(); Navigation.StateChanged += _ => UpdateExecutorState(); _started = true; - Emit("info", "Starting map exploration..."); - State = "Exploring"; - _ = Navigation.RunExploreLoop().ContinueWith(t => + if (Config.MapType == MapType.Kulemak) { - if (t.IsFaulted) + // Boss run needs hideout first + var inHideout = LogWatcher.CurrentArea.Contains("hideout", StringComparison.OrdinalIgnoreCase); + if (!inHideout) { - Log.Error(t.Exception!, "Explore loop failed"); - Emit("error", $"Explore loop failed: {t.Exception?.InnerException?.Message}"); + Emit("info", "Sending /hideout command..."); + var arrivedHome = await Inventory.WaitForAreaTransition(Config.TravelTimeoutMs, () => Game.GoToHideout()); + if (!arrivedHome) + Log.Warning("Timed out waiting for hideout transition on startup"); } - else + Inventory.SetLocation(true); + + Emit("info", "Starting boss run loop..."); + State = "Preparing"; + _ = BossRunExecutor.RunBossLoop().ContinueWith(t => { - Emit("info", "Exploration finished"); - } - State = "Idle"; - StatusUpdated?.Invoke(); - }); + if (t.IsFaulted) + { + Log.Error(t.Exception!, "Boss run loop failed"); + Emit("error", $"Boss run failed: {t.Exception?.InnerException?.Message}"); + } + else + { + Emit("info", "Boss run loop finished"); + } + State = "Idle"; + StatusUpdated?.Invoke(); + }); + } + else + { + Emit("info", "Starting map exploration..."); + State = "Exploring"; + _ = Navigation.RunExploreLoop().ContinueWith(t => + { + if (t.IsFaulted) + { + Log.Error(t.Exception!, "Explore loop failed"); + Emit("error", $"Explore loop failed: {t.Exception?.InnerException?.Message}"); + } + else + { + Emit("info", "Exploration finished"); + } + State = "Idle"; + StatusUpdated?.Invoke(); + }); + } } public async ValueTask DisposeAsync() diff --git a/src/Poe2Trade.Core/ConfigStore.cs b/src/Poe2Trade.Core/ConfigStore.cs index 9f1bc8f..da09e1c 100644 --- a/src/Poe2Trade.Core/ConfigStore.cs +++ b/src/Poe2Trade.Core/ConfigStore.cs @@ -34,6 +34,16 @@ public class SavedSettings public MapType MapType { get; set; } = MapType.TrialOfChaos; public StashCalibration? StashCalibration { get; set; } public StashCalibration? ShopCalibration { get; set; } + public bool ShowHudDebug { get; set; } + public KulemakSettings Kulemak { get; set; } = new(); +} + +public class KulemakSettings +{ + public bool Enabled { get; set; } + public string InvitationTabPath { get; set; } = ""; + public string LootTabPath { get; set; } = ""; + public int InvitationCount { get; set; } = 15; } public class ConfigStore @@ -129,6 +139,33 @@ public class ConfigStore try { var raw = File.ReadAllText(_filePath); + + // Migrate: BossRun was removed from BotMode, now it's MapType.Kulemak + if (raw.Contains("\"bossRun\"") || raw.Contains("\"BossRun\"")) + { + const System.Text.RegularExpressions.RegexOptions ic = + System.Text.RegularExpressions.RegexOptions.IgnoreCase; + + // Mode enum: bossRun → mapping + using var doc = JsonDocument.Parse(raw); + if (doc.RootElement.TryGetProperty("Mode", out var modeProp) && + modeProp.GetString()?.Equals("bossRun", StringComparison.OrdinalIgnoreCase) == true) + { + raw = System.Text.RegularExpressions.Regex.Replace( + raw, @"""Mode""\s*:\s*""bossRun""", @"""Mode"": ""mapping""", ic); + raw = System.Text.RegularExpressions.Regex.Replace( + raw, @"""MapType""\s*:\s*""[^""]*""", @"""MapType"": ""kulemak""", ic); + Log.Information("Migrated config: Mode bossRun -> mapping + MapType kulemak"); + } + + // MapType enum value: bossRun → kulemak + raw = System.Text.RegularExpressions.Regex.Replace( + raw, @"""MapType""\s*:\s*""bossRun""", @"""MapType"": ""kulemak""", ic); + + // Settings property name: BossRun → Kulemak + raw = raw.Replace("\"BossRun\":", "\"Kulemak\":"); + } + var parsed = JsonSerializer.Deserialize(raw, JsonOptions); if (parsed == null) return new SavedSettings(); diff --git a/src/Poe2Trade.Core/Types.cs b/src/Poe2Trade.Core/Types.cs index a05fcc3..f91d79e 100644 --- a/src/Poe2Trade.Core/Types.cs +++ b/src/Poe2Trade.Core/Types.cs @@ -75,11 +75,27 @@ public enum BotMode Mapping } +public enum BossRunState +{ + Idle, + Preparing, + TravelingToZone, + WalkingToEntrance, + UsingInvitation, + Fighting, + Looting, + Returning, + StoringLoot, + Complete, + Failed +} + public enum MapType { TrialOfChaos, Temple, - Endgame + Endgame, + Kulemak } public enum GameUiState diff --git a/src/Poe2Trade.Game/GameController.cs b/src/Poe2Trade.Game/GameController.cs index 06b0725..559d2f9 100644 --- a/src/Poe2Trade.Game/GameController.cs +++ b/src/Poe2Trade.Game/GameController.cs @@ -77,4 +77,10 @@ public class GameController : IGameController public Task ToggleMinimap() => _input.PressKey(InputSender.VK.TAB); public Task KeyDown(int vkCode) => _input.KeyDown(vkCode); public Task KeyUp(int vkCode) => _input.KeyUp(vkCode); + public Task PressPlus() => _input.PressKey(0xBB); // VK_OEM_PLUS + public Task PressKey(int vkCode) => _input.PressKey(vkCode); + public void LeftMouseDown() => _input.LeftMouseDown(); + public void LeftMouseUp() => _input.LeftMouseUp(); + public void RightMouseDown() => _input.RightMouseDown(); + public void RightMouseUp() => _input.RightMouseUp(); } diff --git a/src/Poe2Trade.Game/IGameController.cs b/src/Poe2Trade.Game/IGameController.cs index 36fbbc5..d5e9dfe 100644 --- a/src/Poe2Trade.Game/IGameController.cs +++ b/src/Poe2Trade.Game/IGameController.cs @@ -22,4 +22,10 @@ public interface IGameController Task ToggleMinimap(); Task KeyDown(int vkCode); Task KeyUp(int vkCode); + Task PressPlus(); + Task PressKey(int vkCode); + void LeftMouseDown(); + void LeftMouseUp(); + void RightMouseDown(); + void RightMouseUp(); } diff --git a/src/Poe2Trade.Game/InputSender.cs b/src/Poe2Trade.Game/InputSender.cs index af66dde..c469c97 100644 --- a/src/Poe2Trade.Game/InputSender.cs +++ b/src/Poe2Trade.Game/InputSender.cs @@ -34,6 +34,7 @@ public class InputSender public const int W = 0x57; public const int S = 0x53; public const int D = 0x44; + public const int Z = 0x5A; } public async Task PressKey(int vkCode) @@ -142,6 +143,11 @@ public class InputSender await Helpers.RandomDelay(5, 15); } + public void LeftMouseDown() => SendMouseInput(0, 0, 0, InputNative.MOUSEEVENTF_LEFTDOWN); + public void LeftMouseUp() => SendMouseInput(0, 0, 0, InputNative.MOUSEEVENTF_LEFTUP); + public void RightMouseDown() => SendMouseInput(0, 0, 0, InputNative.MOUSEEVENTF_RIGHTDOWN); + public void RightMouseUp() => SendMouseInput(0, 0, 0, InputNative.MOUSEEVENTF_RIGHTUP); + public void MoveMouseInstant(int x, int y) => MoveMouseRaw(x, y); public async Task MoveMouseFast(int x, int y) diff --git a/src/Poe2Trade.Inventory/IInventoryManager.cs b/src/Poe2Trade.Inventory/IInventoryManager.cs index cedc791..641993b 100644 --- a/src/Poe2Trade.Inventory/IInventoryManager.cs +++ b/src/Poe2Trade.Inventory/IInventoryManager.cs @@ -19,4 +19,6 @@ public interface IInventoryManager Task DepositItemsToStash(List items); Task SalvageItems(List items); (bool[,] Grid, List Items, int Free) GetInventoryState(); + Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null); + Task GrabItemsFromStash(string layoutName, int maxItems, string? templatePath = null); } diff --git a/src/Poe2Trade.Inventory/InventoryManager.cs b/src/Poe2Trade.Inventory/InventoryManager.cs index d36e60a..01a87d4 100644 --- a/src/Poe2Trade.Inventory/InventoryManager.cs +++ b/src/Poe2Trade.Inventory/InventoryManager.cs @@ -155,6 +155,7 @@ public class InventoryManager : IInventoryManager private async Task CtrlClickItems(List items, GridLayout layout, int clickDelayMs = Delays.ClickInterval) { + await _game.KeyDown(Game.InputSender.VK.SHIFT); await _game.HoldCtrl(); foreach (var item in items) { @@ -163,6 +164,7 @@ public class InventoryManager : IInventoryManager await Helpers.Sleep(clickDelayMs); } await _game.ReleaseCtrl(); + await _game.KeyUp(Game.InputSender.VK.SHIFT); await Helpers.Sleep(Delays.PostEscape); } @@ -208,13 +210,31 @@ public class InventoryManager : IInventoryManager for (var attempt = 1; attempt <= maxRetries; attempt++) { Log.Information("Searching for nameplate '{Name}' (attempt {Attempt}/{Max})", name, attempt, maxRetries); - var pos = await _screen.FindTextOnScreen(name, fuzzy: true); + + // Move mouse to bottom-left so it doesn't cover nameplates + _game.MoveMouseInstant(0, 1440); + await Helpers.Sleep(100); + + // Nameplates hidden by default — capture clean reference + using var reference = _screen.CaptureRawBitmap(); + + // Hold Alt to show nameplates, capture, then release + await _game.KeyDown(Game.InputSender.VK.MENU); + await Helpers.Sleep(50); + using var current = _screen.CaptureRawBitmap(); + await _game.KeyUp(Game.InputSender.VK.MENU); + + // Diff OCR — only processes the bright nameplate regions + var result = await _screen.NameplateDiffOcr(reference, current); + var pos = FindWordInOcrResult(result, name, fuzzy: true); if (pos.HasValue) { Log.Information("Clicking nameplate '{Name}' at ({X},{Y})", name, pos.Value.X, pos.Value.Y); await _game.LeftClickAt(pos.Value.X, pos.Value.Y); return pos; } + + Log.Debug("Nameplate '{Name}' not found in diff OCR (attempt {Attempt}), text: {Text}", name, attempt, result.Text); if (attempt < maxRetries) await Helpers.Sleep(retryDelayMs); } @@ -223,6 +243,73 @@ public class InventoryManager : IInventoryManager return null; } + private static (int X, int Y)? FindWordInOcrResult(OcrResponse result, string needle, bool fuzzy) + { + var lower = needle.ToLowerInvariant(); + + // Multi-word: match against full line text + if (lower.Contains(' ')) + { + foreach (var line in result.Lines) + { + if (line.Words.Count == 0) continue; + if (line.Text.Contains(needle, StringComparison.OrdinalIgnoreCase)) + { + var first = line.Words[0]; + var last = line.Words[^1]; + return ((first.X + last.X + last.Width) / 2, (first.Y + last.Y + last.Height) / 2); + } + if (fuzzy) + { + var sim = BigramSimilarity(Normalize(needle), Normalize(line.Text)); + if (sim >= 0.55) + { + var first = line.Words[0]; + var last = line.Words[^1]; + return ((first.X + last.X + last.Width) / 2, (first.Y + last.Y + last.Height) / 2); + } + } + } + return null; + } + + // Single word + foreach (var line in result.Lines) + foreach (var word in line.Words) + { + if (word.Text.Contains(needle, StringComparison.OrdinalIgnoreCase)) + return (word.X + word.Width / 2, word.Y + word.Height / 2); + if (fuzzy && BigramSimilarity(Normalize(needle), Normalize(word.Text)) >= 0.55) + return (word.X + word.Width / 2, word.Y + word.Height / 2); + } + return null; + } + + private static string Normalize(string s) => + new(s.ToLowerInvariant().Where(char.IsLetterOrDigit).ToArray()); + + private static double BigramSimilarity(string a, string b) + { + if (a.Length < 2 || b.Length < 2) return a == b ? 1 : 0; + var bigramsA = new Dictionary<(char, char), int>(); + for (var i = 0; i < a.Length - 1; i++) + { + var bg = (a[i], a[i + 1]); + bigramsA[bg] = bigramsA.GetValueOrDefault(bg) + 1; + } + var matches = 0; + for (var i = 0; i < b.Length - 1; i++) + { + var bg = (b[i], b[i + 1]); + if (bigramsA.TryGetValue(bg, out var count) && count > 0) + { + matches++; + bigramsA[bg] = count - 1; + } + } + return 2.0 * matches / (a.Length - 1 + b.Length - 1); + } + public async Task WaitForAreaTransition(int timeoutMs, Func? triggerAction = null) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); @@ -254,4 +341,64 @@ public class InventoryManager : IInventoryManager { return (Tracker.GetGrid(), Tracker.GetItems(), Tracker.FreeCells); } + + public async Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null) + { + if (parentFolder != null) + { + Log.Information("Clicking folder '{Folder}' at ({X},{Y})", parentFolder.Name, parentFolder.ClickX, parentFolder.ClickY); + await _game.LeftClickAt(parentFolder.ClickX, parentFolder.ClickY); + await Helpers.Sleep(200); + } + + Log.Information("Clicking tab '{Tab}' at ({X},{Y})", tab.Name, tab.ClickX, tab.ClickY); + await _game.LeftClickAt(tab.ClickX, tab.ClickY); + await Helpers.Sleep(Delays.PostStashOpen); + } + + public async Task GrabItemsFromStash(string layoutName, int maxItems, string? templatePath = null) + { + Log.Information("Grabbing up to {Max} items from stash layout '{Layout}' (template={Template})", + maxItems, layoutName, templatePath ?? "none"); + + var layout = GridLayouts.All[layoutName]; + + if (templatePath != null) + { + // Template matching mode: repeatedly find and click matching items + var grabbed = 0; + await _game.HoldCtrl(); + while (grabbed < maxItems) + { + var match = await _screen.TemplateMatch(templatePath, layout.Region); + if (match == null) break; + + Log.Information("Template match at ({X},{Y}), grabbing", match.X, match.Y); + await _game.LeftClickAt(match.X, match.Y); + await Helpers.Sleep(Delays.ClickInterval); + grabbed++; + } + await _game.ReleaseCtrl(); + await Helpers.Sleep(Delays.PostEscape); + Log.Information("Grabbed {Count} matching items from stash", grabbed); + } + else + { + // Grid scan mode: grab all occupied cells + var result = await _screen.Grid.Scan(layoutName); + var grabbed = 0; + await _game.HoldCtrl(); + foreach (var cell in result.Occupied) + { + if (grabbed >= maxItems) break; + var center = _screen.Grid.GetCellCenter(layout, cell.Row, cell.Col); + await _game.LeftClickAt(center.X, center.Y); + await Helpers.Sleep(Delays.ClickInterval); + grabbed++; + } + await _game.ReleaseCtrl(); + await Helpers.Sleep(Delays.PostEscape); + Log.Information("Grabbed {Count} items from stash", grabbed); + } + } } diff --git a/src/Poe2Trade.Screen/BossDetector.cs b/src/Poe2Trade.Screen/BossDetector.cs new file mode 100644 index 0000000..6ea8fd4 --- /dev/null +++ b/src/Poe2Trade.Screen/BossDetector.cs @@ -0,0 +1,78 @@ +using Poe2Trade.Core; +using Serilog; +using Region = Poe2Trade.Core.Region; + +namespace Poe2Trade.Screen; + +public class BossDetector : IFrameConsumer, IDisposable +{ + private const int DetectEveryNFrames = 6; + private const int MinConsecutiveFrames = 2; + + private readonly PythonDetectBridge _bridge = new(); + private volatile BossSnapshot _latest = new([], 0, 0); + private int _frameCounter; + private int _consecutiveDetections; + private string _modelName = "boss-kulemak"; + + public bool Enabled { get; set; } + public BossSnapshot Latest => _latest; + public event Action? BossDetected; + + public void SetBoss(string bossName) + { + _modelName = $"boss-{bossName}"; + _consecutiveDetections = 0; + } + + public void Process(ScreenFrame frame) + { + if (!Enabled) return; + if (++_frameCounter % DetectEveryNFrames != 0) return; + + try + { + // Use full frame — model was trained on full 2560x1440 screenshots + var fullRegion = new Region(0, 0, frame.Width, frame.Height); + using var bgr = frame.CropBgr(fullRegion); + var result = _bridge.Detect(bgr, conf: 0.60f, imgsz: 1280, model: _modelName); + + var bosses = new List(result.Count); + foreach (var det in result.Detections) + { + bosses.Add(new DetectedBoss( + det.ClassName, + det.Confidence, + det.X, + det.Y, + det.Width, + det.Height, + det.Cx, + det.Cy)); + } + + var snapshot = new BossSnapshot( + bosses.AsReadOnly(), + DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + result.InferenceMs); + + _latest = snapshot; + if (bosses.Count > 0) + { + _consecutiveDetections++; + if (_consecutiveDetections >= MinConsecutiveFrames) + BossDetected?.Invoke(snapshot); + } + else + { + _consecutiveDetections = 0; + } + } + catch (Exception ex) + { + Log.Debug(ex, "BossDetector YOLO failed"); + } + } + + public void Dispose() => _bridge.Dispose(); +} diff --git a/src/Poe2Trade.Screen/DetectionTypes.cs b/src/Poe2Trade.Screen/DetectionTypes.cs index f76ef8c..6bb9b9f 100644 --- a/src/Poe2Trade.Screen/DetectionTypes.cs +++ b/src/Poe2Trade.Screen/DetectionTypes.cs @@ -10,3 +10,14 @@ public record DetectionSnapshot( IReadOnlyList Enemies, long Timestamp, float InferenceMs); + +public record DetectedBoss( + string ClassName, + float Confidence, + int X, int Y, int Width, int Height, + int Cx, int Cy); + +public record BossSnapshot( + IReadOnlyList Bosses, + long Timestamp, + float InferenceMs); diff --git a/src/Poe2Trade.Screen/FrameSaver.cs b/src/Poe2Trade.Screen/FrameSaver.cs index 65ad6bc..9bdfb17 100644 --- a/src/Poe2Trade.Screen/FrameSaver.cs +++ b/src/Poe2Trade.Screen/FrameSaver.cs @@ -16,6 +16,7 @@ public class FrameSaver : IFrameConsumer private const int JpegQuality = 95; private const int MinSaveIntervalMs = 1000; + private const int BurstIntervalMs = 200; private const int MinRedPixels = 50; private const int ThumbSize = 64; private const double MovementThreshold = 8.0; // mean absolute diff on 64x64 grayscale @@ -26,6 +27,7 @@ public class FrameSaver : IFrameConsumer private Mat? _prevThumb; public bool Enabled { get; set; } + public bool BurstMode { get; set; } public int SavedCount => _savedCount; public FrameSaver(string outputDir = "training-data/raw") @@ -35,10 +37,11 @@ public class FrameSaver : IFrameConsumer public void Process(ScreenFrame frame) { - if (!Enabled) return; + if (!Enabled && !BurstMode) return; var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); - if (now - _lastSaveTime < MinSaveIntervalMs) return; + var interval = BurstMode ? BurstIntervalMs : MinSaveIntervalMs; + if (now - _lastSaveTime < interval) return; if (GameplayRegion.X + GameplayRegion.Width > frame.Width || GameplayRegion.Y + GameplayRegion.Height > frame.Height) @@ -46,10 +49,12 @@ public class FrameSaver : IFrameConsumer try { - using var bgr = frame.CropBgr(GameplayRegion); - - if (!HasHealthBars(bgr)) return; - if (!HasSceneChanged(bgr)) return; + if (!BurstMode) + { + using var bgr = frame.CropBgr(GameplayRegion); + if (!HasHealthBars(bgr)) return; + if (!HasSceneChanged(bgr)) return; + } if (!Directory.Exists(_outputDir)) Directory.CreateDirectory(_outputDir); diff --git a/src/Poe2Trade.Screen/HudReader.cs b/src/Poe2Trade.Screen/HudReader.cs index ef7769b..fb867a8 100644 --- a/src/Poe2Trade.Screen/HudReader.cs +++ b/src/Poe2Trade.Screen/HudReader.cs @@ -1,5 +1,3 @@ -using System.Drawing; -using System.Text.RegularExpressions; using OpenCvSharp; using Poe2Trade.Core; using Serilog; @@ -7,38 +5,41 @@ using Region = Poe2Trade.Core.Region; namespace Poe2Trade.Screen; -public record HudValues(int Current, int Max); - public record HudSnapshot { - public HudValues? Life { get; init; } - public HudValues? Mana { get; init; } - public HudValues? EnergyShield { get; init; } - public HudValues? Spirit { get; init; } + public float LifePct { get; init; } + public float ShieldPct { get; init; } + public float ManaPct { get; init; } public long Timestamp { get; init; } - - public float LifePct => Life is { Max: > 0 } l ? (float)l.Current / l.Max : 1f; - public float ManaPct => Mana is { Max: > 0 } m ? (float)m.Current / m.Max : 1f; } /// -/// Reads life/mana/ES/spirit values from HUD globe text via OCR. -/// Throttled to ~1 read per second (every 30 frames at 30fps). +/// Reads life/mana/shield fill levels by sampling pixel colors on the globes. +/// Finds the highest Y where the fill color appears — the fill drains from top down. +/// Samples a horizontal band (±SampleHalfWidth) at each Y for robustness against the frame ornaments. /// public class HudReader : IFrameConsumer { - private static readonly Regex ValuePattern = new(@"(\d+)\s*/\s*(\d+)", RegexOptions.Compiled); + // Globe centers at 2560x1440 + private const int LifeX = 167; + private const int ManaX = 2394; + private const int GlobeTop = 1185; + private const int GlobeBottom = 1411; - // Crop regions for HUD text at 2560x1440 — placeholders, need calibration - private static readonly Region LifeRegion = new(100, 1340, 200, 40); - private static readonly Region ManaRegion = new(2260, 1340, 200, 40); - private static readonly Region EsRegion = new(100, 1300, 200, 40); - private static readonly Region SpiritRegion = new(2260, 1300, 200, 40); + // Shield ring: circle centered at (168, 1294), radius 130 + private const int ShieldCX = 170; + private const int ShieldCY = 1298; + private const int ShieldRadius = 130; - private const int OcrEveryNFrames = 30; + // Sample a horizontal band of pixels at each Y level + private const int SampleHalfWidth = 8; + // Minimum pixels in the band that must match to count as "filled" + private const int MinHits = 2; - private readonly PythonOcrBridge _ocr = new(); - private volatile HudSnapshot _current = new() { Timestamp = 0 }; + private const int MinChannel = 60; + private const float DominanceRatio = 1.2f; + + private volatile HudSnapshot _current = new(); private int _frameCounter; public HudSnapshot Current => _current; @@ -47,64 +48,128 @@ public class HudReader : IFrameConsumer public void Process(ScreenFrame frame) { - if (++_frameCounter % OcrEveryNFrames != 0) return; + if (++_frameCounter % 2 != 0) return; try { - var life = ReadValue(frame, LifeRegion); - var mana = ReadValue(frame, ManaRegion); - var es = ReadValue(frame, EsRegion); - var spirit = ReadValue(frame, SpiritRegion); + var manaPct = SampleFillLevel(frame, ManaX, IsManaPixel); + var shieldPct = SampleShieldRing(frame); + + // If life globe is cyan (1-life build), life = 0 + var redFill = SampleFillLevel(frame, LifeX, IsLifePixel); + var cyanFill = SampleFillLevel(frame, LifeX, IsCyanPixel); + var lifePct = cyanFill > redFill ? 0f : redFill; var snapshot = new HudSnapshot { - Life = life, - Mana = mana, - EnergyShield = es, - Spirit = spirit, + LifePct = lifePct, + ManaPct = manaPct, + ShieldPct = shieldPct, Timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), }; _current = snapshot; Updated?.Invoke(snapshot); - if (snapshot.LifePct < 0.3f) + if (lifePct < 0.3f) LowLife?.Invoke(snapshot); } catch (Exception ex) { - Log.Debug(ex, "HudReader OCR failed"); + Log.Debug(ex, "HudReader sample failed"); } } - private HudValues? ReadValue(ScreenFrame frame, Region region) + /// + /// Scan from top to bottom to find the first Y row where the fill color appears. + /// Fill % = 1 - (firstFilledY - GlobeTop) / (GlobeBottom - GlobeTop). + /// At each Y, sample a horizontal band of pixels and require MinHits matches. + /// + private static float SampleFillLevel(ScreenFrame frame, int centerX, Func colorTest) { - // Bounds check - if (region.X + region.Width > frame.Width || region.Y + region.Height > frame.Height) - return null; + if (centerX >= frame.Width || GlobeBottom >= frame.Height) return 0f; - using var bgr = frame.CropBgr(region); - using var gray = new Mat(); - Cv2.CvtColor(bgr, gray, ColorConversionCodes.BGR2GRAY); + int height = GlobeBottom - GlobeTop; + if (height <= 0) return 0f; - // Threshold for white text on dark background - using var thresh = new Mat(); - Cv2.Threshold(gray, thresh, 180, 255, ThresholdTypes.Binary); + int xMin = Math.Max(0, centerX - SampleHalfWidth); + int xMax = Math.Min(frame.Width - 1, centerX + SampleHalfWidth); - // Convert to Bitmap for OCR bridge - var bytes = thresh.ToBytes(".png"); - using var ms = new System.IO.MemoryStream(bytes); - using var bitmap = new Bitmap(ms); + // Scan from top down — find first row with enough matching pixels + for (int y = GlobeTop; y <= GlobeBottom; y++) + { + int hits = 0; + for (int x = xMin; x <= xMax; x++) + { + if (colorTest(frame.PixelAt(x, y))) + hits++; + if (hits >= MinHits) break; + } - var result = _ocr.OcrFromBitmap(bitmap); - if (string.IsNullOrWhiteSpace(result.Text)) return null; + if (hits >= MinHits) + { + // Fill level = how far down from top this first row is + // If found at GlobeTop → 100%, at GlobeBottom → 0% + return 1f - (float)(y - GlobeTop) / height; + } + } - var match = ValuePattern.Match(result.Text); - if (!match.Success) return null; - - return new HudValues( - int.Parse(match.Groups[1].Value), - int.Parse(match.Groups[2].Value) - ); + return 0f; // no fill found } + + /// + /// Sample the shield ring — right semicircle (12 o'clock to 6 o'clock) around the life globe. + /// Scans from bottom (6 o'clock) upward along the arc, tracking contiguous cyan fill. + /// + private static float SampleShieldRing(ScreenFrame frame) + { + int yTop = ShieldCY - ShieldRadius; + int yBot = ShieldCY + ShieldRadius; + if (yBot >= frame.Height) return 0f; + + int r2 = ShieldRadius * ShieldRadius; + + // Scan from top (12 o'clock) down along the right arc + // When we find the first cyan row, convert Y to arc fraction + for (int y = yTop; y <= yBot; y++) + { + int dy = y - ShieldCY; + int dx = (int)Math.Sqrt(r2 - dy * dy); + int arcX = ShieldCX + dx; + + if (arcX >= frame.Width) continue; + + int hits = 0; + for (int x = Math.Max(0, arcX - 3); x <= Math.Min(frame.Width - 1, arcX + 3); x++) + { + if (IsCyanPixel(frame.PixelAt(x, y))) + hits++; + if (hits >= MinHits) break; + } + + if (hits >= MinHits) + { + // Convert Y to angle on the semicircle: θ = arcsin((y - cy) / r) + // Arc fraction from top = (θ + π/2) / π + // Fill = 1 - arc_fraction + var theta = Math.Asin(Math.Clamp((double)(y - ShieldCY) / ShieldRadius, -1, 1)); + var arcFraction = (theta + Math.PI / 2) / Math.PI; + return (float)(1.0 - arcFraction); + } + } + + return 0f; + } + + // B=0, G=1, R=2, A=3 + private static bool IsLifePixel(Vec4b px) => + px[2] > MinChannel && px[2] > px[1] * DominanceRatio && px[2] > px[0] * DominanceRatio; + + private static bool IsManaPixel(Vec4b px) => + px[0] > MinChannel && px[0] > px[1] * DominanceRatio && px[0] > px[2] * DominanceRatio; + + private static bool IsCyanPixel(Vec4b px) => + px[0] > MinChannel && px[1] > MinChannel + && px[0] > px[2] * DominanceRatio + && px[1] > px[2] * DominanceRatio; } diff --git a/src/Poe2Trade.Screen/IScreenReader.cs b/src/Poe2Trade.Screen/IScreenReader.cs index 3293991..b9b9b95 100644 --- a/src/Poe2Trade.Screen/IScreenReader.cs +++ b/src/Poe2Trade.Screen/IScreenReader.cs @@ -17,6 +17,8 @@ public interface IScreenReader : IDisposable Task Snapshot(); Task DiffOcr(string? savePath = null, Region? region = null); Task TemplateMatch(string templatePath, Region? region = null); + Task NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current); + System.Drawing.Bitmap CaptureRawBitmap(); Task SaveScreenshot(string path); Task SaveRegion(Region region, string path); } diff --git a/src/Poe2Trade.Screen/PythonDetectBridge.cs b/src/Poe2Trade.Screen/PythonDetectBridge.cs index 7a398a3..c262cf1 100644 --- a/src/Poe2Trade.Screen/PythonDetectBridge.cs +++ b/src/Poe2Trade.Screen/PythonDetectBridge.cs @@ -35,7 +35,7 @@ class PythonDetectBridge : IDisposable /// /// Run YOLO detection on a BGR Mat. Returns parsed detection results. /// - public DetectResponse Detect(Mat bgrMat, float conf = 0.3f, float iou = 0.45f, int imgsz = 640) + public DetectResponse Detect(Mat bgrMat, float conf = 0.3f, float iou = 0.45f, int imgsz = 640, string? model = null) { EnsureRunning(); @@ -49,6 +49,7 @@ class PythonDetectBridge : IDisposable ["conf"] = conf, ["iou"] = iou, ["imgsz"] = imgsz, + ["model"] = model, }; return SendRequest(req); diff --git a/src/Poe2Trade.Screen/ScreenReader.cs b/src/Poe2Trade.Screen/ScreenReader.cs index 11c53d9..77f6d82 100644 --- a/src/Poe2Trade.Screen/ScreenReader.cs +++ b/src/Poe2Trade.Screen/ScreenReader.cs @@ -1,6 +1,10 @@ +using System.Drawing; +using System.Drawing.Imaging; +using System.Runtime.InteropServices; using Poe2Trade.Core; using OpenCvSharp.Extensions; using Serilog; +using Region = Poe2Trade.Core.Region; namespace Poe2Trade.Screen; @@ -178,6 +182,144 @@ public class ScreenReader : IScreenReader return Task.CompletedTask; } + // -- Nameplate diff OCR -- + + public Bitmap CaptureRawBitmap() => ScreenCapture.CaptureOrLoad(null, null); + + public Task NameplateDiffOcr(Bitmap reference, Bitmap current) + { + int w = Math.Min(reference.Width, current.Width); + int h = Math.Min(reference.Height, current.Height); + + var refData = reference.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); + var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); + byte[] refPx = new byte[refData.Stride * h]; + byte[] curPx = new byte[curData.Stride * h]; + Marshal.Copy(refData.Scan0, refPx, 0, refPx.Length); + Marshal.Copy(curData.Scan0, curPx, 0, curPx.Length); + int stride = refData.Stride; + reference.UnlockBits(refData); + current.UnlockBits(curData); + + // Build a binary mask of pixels that got significantly brighter (nameplates are bright text) + const int brightThresh = 30; + bool[] mask = new bool[w * h]; + Parallel.For(0, h, y => + { + int rowOff = y * stride; + for (int x = 0; x < w; x++) + { + int i = rowOff + x * 4; + int brighter = (curPx[i] - refPx[i]) + (curPx[i + 1] - refPx[i + 1]) + (curPx[i + 2] - refPx[i + 2]); + if (brighter > brightThresh) + mask[y * w + x] = true; + } + }); + + // Find connected clusters via row-scan: collect bounding boxes of bright regions + var boxes = FindBrightClusters(mask, w, h, minWidth: 40, minHeight: 10, maxGap: 8); + Log.Information("NameplateDiff: found {Count} bright clusters", boxes.Count); + + if (boxes.Count == 0) + return Task.FromResult(new OcrResponse { Text = "", Lines = [] }); + + // OCR each cluster crop, accumulate results with screen-space coordinates + var allLines = new List(); + var allText = new List(); + + foreach (var box in boxes) + { + // Pad the crop slightly + int pad = 4; + int cx = Math.Max(0, box.X - pad); + int cy = Math.Max(0, box.Y - pad); + int cw = Math.Min(w - cx, box.Width + pad * 2); + int ch = Math.Min(h - cy, box.Height + pad * 2); + + using var crop = current.Clone(new Rectangle(cx, cy, cw, ch), PixelFormat.Format32bppArgb); + var ocrResult = _pythonBridge.OcrFromBitmap(crop); + + // Offset word coordinates to screen space + foreach (var line in ocrResult.Lines) + { + foreach (var word in line.Words) + { + word.X += cx; + word.Y += cy; + } + allLines.Add(line); + allText.Add(line.Text); + } + } + + return Task.FromResult(new OcrResponse + { + Text = string.Join("\n", allText), + Lines = allLines, + }); + } + + private static List FindBrightClusters(bool[] mask, int w, int h, int minWidth, int minHeight, int maxGap) + { + // Row density + int[] rowCounts = new int[h]; + for (int y = 0; y < h; y++) + for (int x = 0; x < w; x++) + if (mask[y * w + x]) rowCounts[y]++; + + // Find horizontal bands of bright rows + int rowThresh = 3; + var bands = new List<(int Top, int Bottom)>(); + int bandStart = -1, lastActive = -1; + for (int y = 0; y < h; y++) + { + if (rowCounts[y] >= rowThresh) + { + if (bandStart < 0) bandStart = y; + lastActive = y; + } + else if (bandStart >= 0 && y - lastActive > maxGap) + { + if (lastActive - bandStart + 1 >= minHeight) + bands.Add((bandStart, lastActive)); + bandStart = -1; + } + } + if (bandStart >= 0 && lastActive - bandStart + 1 >= minHeight) + bands.Add((bandStart, lastActive)); + + // For each band, find column extents to get individual nameplate boxes + var boxes = new List(); + foreach (var (top, bottom) in bands) + { + int[] colCounts = new int[w]; + for (int y = top; y <= bottom; y++) + for (int x = 0; x < w; x++) + if (mask[y * w + x]) colCounts[x]++; + + int colThresh = 1; + int colStart = -1, lastCol = -1; + for (int x = 0; x < w; x++) + { + if (colCounts[x] >= colThresh) + { + if (colStart < 0) colStart = x; + lastCol = x; + } + else if (colStart >= 0 && x - lastCol > maxGap) + { + if (lastCol - colStart + 1 >= minWidth) + boxes.Add(new Rectangle(colStart, top, lastCol - colStart + 1, bottom - top + 1)); + colStart = -1; + } + } + if (colStart >= 0 && lastCol - colStart + 1 >= minWidth) + boxes.Add(new Rectangle(colStart, top, lastCol - colStart + 1, bottom - top + 1)); + } + + return boxes; + } + public void Dispose() => _pythonBridge.Dispose(); // -- OCR text matching -- diff --git a/src/Poe2Trade.Screen/TemplateMatchHandler.cs b/src/Poe2Trade.Screen/TemplateMatchHandler.cs index 3aef223..d030eb6 100644 --- a/src/Poe2Trade.Screen/TemplateMatchHandler.cs +++ b/src/Poe2Trade.Screen/TemplateMatchHandler.cs @@ -27,27 +27,56 @@ class TemplateMatchHandler else screenMat.CopyTo(screenBgr); - // Template must fit within screenshot - if (template.Rows > screenBgr.Rows || template.Cols > screenBgr.Cols) + // Try exact size first (fast path) + var exact = MatchAtScale(screenBgr, template, region, 1.0, threshold); + if (exact is { Confidence: > 0.95 }) + return exact; + + // Multi-scale: resize template from 50% to 150% in steps of 10% + TemplateMatchResult? best = exact; + for (var pct = 50; pct <= 150; pct += 10) + { + var scale = pct / 100.0; + if (pct == 100) continue; // already tried + + var match = MatchAtScale(screenBgr, template, region, scale, threshold); + if (match != null && (best == null || match.Confidence > best.Confidence)) + { + best = match; + if (best.Confidence > 0.95) break; + } + } + + return best; + } + + private static TemplateMatchResult? MatchAtScale(Mat screen, Mat template, + Region? region, double scale, double threshold) + { + using var scaled = scale == 1.0 ? template.Clone() + : template.Resize(new OpenCvSharp.Size( + Math.Max(1, (int)(template.Cols * scale)), + Math.Max(1, (int)(template.Rows * scale)))); + + if (scaled.Rows > screen.Rows || scaled.Cols > screen.Cols) return null; using var result = new Mat(); - Cv2.MatchTemplate(screenBgr, template, result, TemplateMatchModes.CCoeffNormed); - + Cv2.MatchTemplate(screen, scaled, result, TemplateMatchModes.CCoeffNormed); Cv2.MinMaxLoc(result, out _, out double maxVal, out _, out OpenCvSharp.Point maxLoc); if (maxVal < threshold) return null; - int offsetX = region?.X ?? 0; - int offsetY = region?.Y ?? 0; + var offsetX = region?.X ?? 0; + var offsetY = region?.Y ?? 0; return new TemplateMatchResult { - X = offsetX + maxLoc.X + template.Cols / 2, - Y = offsetY + maxLoc.Y + template.Rows / 2, - Width = template.Cols, - Height = template.Rows, + X = offsetX + maxLoc.X + scaled.Cols / 2, + Y = offsetY + maxLoc.Y + scaled.Rows / 2, + Width = scaled.Cols, + Height = scaled.Rows, Confidence = maxVal, }; } diff --git a/src/Poe2Trade.Ui/Converters/ValueConverters.cs b/src/Poe2Trade.Ui/Converters/ValueConverters.cs index e6de743..f9db3f2 100644 --- a/src/Poe2Trade.Ui/Converters/ValueConverters.cs +++ b/src/Poe2Trade.Ui/Converters/ValueConverters.cs @@ -103,6 +103,7 @@ public class MapRequirementsConverter : IValueConverter MapType.TrialOfChaos => "Trial Token x1", MapType.Temple => "Identity Scroll x20", MapType.Endgame => "Identity Scroll x20", + MapType.Kulemak => "Invitation x1", _ => "", }; } diff --git a/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs b/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs index cd2a697..dd11c49 100644 --- a/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs +++ b/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs @@ -181,13 +181,16 @@ public sealed class D2dOverlay private OverlayState BuildState(double fps, RenderTiming timing) { var detection = _bot.EnemyDetector.Latest; + var bossDetection = _bot.BossDetector.Latest; return new OverlayState( Enemies: detection.Enemies, + Bosses: bossDetection.Bosses, InferenceMs: detection.InferenceMs, Hud: _bot.HudReader.Current, NavState: _bot.Navigation.State, NavPosition: _bot.Navigation.Position, IsExploring: _bot.Navigation.IsExploring, + ShowHudDebug: _bot.Store.Settings.ShowHudDebug, Fps: fps, Timing: timing); } diff --git a/src/Poe2Trade.Ui/Overlay/D2dRenderContext.cs b/src/Poe2Trade.Ui/Overlay/D2dRenderContext.cs index a8f9ca7..178201e 100644 --- a/src/Poe2Trade.Ui/Overlay/D2dRenderContext.cs +++ b/src/Poe2Trade.Ui/Overlay/D2dRenderContext.cs @@ -24,11 +24,13 @@ public sealed class D2dRenderContext : IDisposable // Pre-created brushes public ID2D1SolidColorBrush Red { get; private set; } = null!; public ID2D1SolidColorBrush Yellow { get; private set; } = null!; + public ID2D1SolidColorBrush Cyan { get; private set; } = null!; public ID2D1SolidColorBrush Green { get; private set; } = null!; public ID2D1SolidColorBrush White { get; private set; } = null!; public ID2D1SolidColorBrush Gray { get; private set; } = null!; public ID2D1SolidColorBrush LifeBrush { get; private set; } = null!; public ID2D1SolidColorBrush ManaBrush { get; private set; } = null!; + public ID2D1SolidColorBrush ShieldBrush { get; private set; } = null!; public ID2D1SolidColorBrush BarBgBrush { get; private set; } = null!; public ID2D1SolidColorBrush LabelBgBrush { get; private set; } = null!; public ID2D1SolidColorBrush DebugTextBrush { get; private set; } = null!; @@ -79,11 +81,13 @@ public sealed class D2dRenderContext : IDisposable { Red = RenderTarget.CreateSolidColorBrush(new Color4(1f, 0f, 0f, 1f)); Yellow = RenderTarget.CreateSolidColorBrush(new Color4(1f, 1f, 0f, 1f)); + Cyan = RenderTarget.CreateSolidColorBrush(new Color4(0f, 1f, 1f, 1f)); Green = RenderTarget.CreateSolidColorBrush(new Color4(0.31f, 1f, 0.31f, 1f)); // 80,255,80 White = RenderTarget.CreateSolidColorBrush(new Color4(1f, 1f, 1f, 1f)); Gray = RenderTarget.CreateSolidColorBrush(new Color4(0.5f, 0.5f, 0.5f, 1f)); LifeBrush = RenderTarget.CreateSolidColorBrush(new Color4(200 / 255f, 40 / 255f, 40 / 255f, 1f)); ManaBrush = RenderTarget.CreateSolidColorBrush(new Color4(40 / 255f, 80 / 255f, 200 / 255f, 1f)); + ShieldBrush = RenderTarget.CreateSolidColorBrush(new Color4(100 / 255f, 180 / 255f, 220 / 255f, 1f)); BarBgBrush = RenderTarget.CreateSolidColorBrush(new Color4(20 / 255f, 20 / 255f, 20 / 255f, 140 / 255f)); LabelBgBrush = RenderTarget.CreateSolidColorBrush(new Color4(0f, 0f, 0f, 160 / 255f)); DebugTextBrush = RenderTarget.CreateSolidColorBrush(new Color4(80 / 255f, 1f, 80 / 255f, 1f)); @@ -95,11 +99,13 @@ public sealed class D2dRenderContext : IDisposable { Red?.Dispose(); Yellow?.Dispose(); + Cyan?.Dispose(); Green?.Dispose(); White?.Dispose(); Gray?.Dispose(); LifeBrush?.Dispose(); ManaBrush?.Dispose(); + ShieldBrush?.Dispose(); BarBgBrush?.Dispose(); LabelBgBrush?.Dispose(); DebugTextBrush?.Dispose(); diff --git a/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs b/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs index ce824cd..402742a 100644 --- a/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs +++ b/src/Poe2Trade.Ui/Overlay/IOverlayLayer.cs @@ -5,11 +5,13 @@ namespace Poe2Trade.Ui.Overlay; public record OverlayState( IReadOnlyList Enemies, + IReadOnlyList Bosses, float InferenceMs, HudSnapshot? Hud, NavigationState NavState, MapPosition NavPosition, bool IsExploring, + bool ShowHudDebug, double Fps, RenderTiming? Timing); diff --git a/src/Poe2Trade.Ui/Overlay/Layers/D2dDebugTextLayer.cs b/src/Poe2Trade.Ui/Overlay/Layers/D2dDebugTextLayer.cs index 2afcdc0..6f8d94b 100644 --- a/src/Poe2Trade.Ui/Overlay/Layers/D2dDebugTextLayer.cs +++ b/src/Poe2Trade.Ui/Overlay/Layers/D2dDebugTextLayer.cs @@ -27,7 +27,7 @@ internal sealed class D2dDebugTextLayer : ID2dOverlayLayer, IDisposable UpdateCache(ctx, _left, ref lc, $"Pos: ({state.NavPosition.X:F0}, {state.NavPosition.Y:F0})", ctx.DebugTextBrush); UpdateCache(ctx, _left, ref lc, $"Enemies: {state.Enemies.Count} YOLO: {state.InferenceMs:F1}ms", ctx.DebugTextBrush); if (state.Hud is { Timestamp: > 0 } hud) - UpdateCache(ctx, _left, ref lc, $"HP: {hud.LifePct:P0} MP: {hud.ManaPct:P0}", ctx.DebugTextBrush); + UpdateCache(ctx, _left, ref lc, $"HP: {hud.LifePct:P0} ES: {hud.ShieldPct:P0} MP: {hud.ManaPct:P0}", ctx.DebugTextBrush); // Right column: timing if (state.Timing != null) diff --git a/src/Poe2Trade.Ui/Overlay/Layers/D2dEnemyBoxLayer.cs b/src/Poe2Trade.Ui/Overlay/Layers/D2dEnemyBoxLayer.cs index 584e88a..48ea2c0 100644 --- a/src/Poe2Trade.Ui/Overlay/Layers/D2dEnemyBoxLayer.cs +++ b/src/Poe2Trade.Ui/Overlay/Layers/D2dEnemyBoxLayer.cs @@ -11,8 +11,13 @@ internal sealed class D2dEnemyBoxLayer : ID2dOverlayLayer, IDisposable private readonly IDWriteTextLayout[] _confirmedLabels = new IDWriteTextLayout[101]; private readonly IDWriteTextLayout[] _unconfirmedLabels = new IDWriteTextLayout[101]; + // Boss labels: cached by "classname NN%" string + private readonly Dictionary _bossLabels = new(); + private readonly D2dRenderContext _ctx; + public D2dEnemyBoxLayer(D2dRenderContext ctx) { + _ctx = ctx; for (int i = 0; i <= 100; i++) { var text = $"{i}%"; @@ -41,18 +46,43 @@ internal sealed class D2dEnemyBoxLayer : ID2dOverlayLayer, IDisposable var labelX = enemy.X; var labelY = enemy.Y - m.Height - 2; - // Background behind label rt.FillRectangle( new RectangleF(labelX - 1, labelY - 1, m.Width + 2, m.Height + 2), ctx.LabelBgBrush); rt.DrawTextLayout(new System.Numerics.Vector2(labelX, labelY), layout, textBrush); } + + // Boss bounding boxes (cyan) + foreach (var boss in state.Bosses) + { + var rect = new RectangleF(boss.X, boss.Y, boss.Width, boss.Height); + rt.DrawRectangle(rect, ctx.Cyan, 3f); + + var pct = Math.Clamp((int)(boss.Confidence * 100), 0, 100); + var key = $"{boss.ClassName} {pct}%"; + if (!_bossLabels.TryGetValue(key, out var layout)) + { + layout = _ctx.CreateTextLayout(key, _ctx.LabelFormat); + _bossLabels[key] = layout; + } + + var m = layout.Metrics; + var labelX = boss.X; + var labelY = boss.Y - m.Height - 2; + + rt.FillRectangle( + new RectangleF(labelX - 1, labelY - 1, m.Width + 2, m.Height + 2), + ctx.LabelBgBrush); + + rt.DrawTextLayout(new System.Numerics.Vector2(labelX, labelY), layout, ctx.Cyan); + } } public void Dispose() { foreach (var l in _confirmedLabels) l?.Dispose(); foreach (var l in _unconfirmedLabels) l?.Dispose(); + foreach (var l in _bossLabels.Values) l?.Dispose(); } } diff --git a/src/Poe2Trade.Ui/Overlay/Layers/D2dHudInfoLayer.cs b/src/Poe2Trade.Ui/Overlay/Layers/D2dHudInfoLayer.cs index 0c89b53..8ce549c 100644 --- a/src/Poe2Trade.Ui/Overlay/Layers/D2dHudInfoLayer.cs +++ b/src/Poe2Trade.Ui/Overlay/Layers/D2dHudInfoLayer.cs @@ -7,15 +7,20 @@ namespace Poe2Trade.Ui.Overlay.Layers; internal sealed class D2dHudInfoLayer : ID2dOverlayLayer, IDisposable { - private const float BarWidth = 200; + private const float BarWidth = 160; private const float BarHeight = 16; - private const float BarY = 1300; - private const float LifeBarX = 1130; - private const float ManaBarX = 1230; + private const float BarGap = 8; + private const float BarY = 1416; // near bottom of 1440 + + // 3 bars centered: total = 160*3 + 8*2 = 496, start = (2560-496)/2 = 1032 + private const float LifeBarX = 1032; + private const float ShieldBarX = LifeBarX + BarWidth + BarGap; + private const float ManaBarX = ShieldBarX + BarWidth + BarGap; - // Cached bar value layouts private string? _lifeLabel; private IDWriteTextLayout? _lifeLayout; + private string? _shieldLabel; + private IDWriteTextLayout? _shieldLayout; private string? _manaLabel; private IDWriteTextLayout? _manaLayout; @@ -23,14 +28,24 @@ internal sealed class D2dHudInfoLayer : ID2dOverlayLayer, IDisposable { if (state.Hud == null || state.Hud.Timestamp == 0) return; - DrawBar(ctx, LifeBarX, BarY, state.Hud.LifePct, ctx.LifeBrush, state.Hud.Life, + DrawBar(ctx, LifeBarX, BarY, state.Hud.LifePct, ctx.LifeBrush, ref _lifeLabel, ref _lifeLayout); - DrawBar(ctx, ManaBarX, BarY, state.Hud.ManaPct, ctx.ManaBrush, state.Hud.Mana, + DrawBar(ctx, ShieldBarX, BarY, state.Hud.ShieldPct, ctx.ShieldBrush, + ref _shieldLabel, ref _shieldLayout); + DrawBar(ctx, ManaBarX, BarY, state.Hud.ManaPct, ctx.ManaBrush, ref _manaLabel, ref _manaLayout); + + // DEBUG: draw sampling lines + if (state.ShowHudDebug) + { + DrawShieldArc(ctx); + DrawSampleLine(ctx, 167, 1185, 1411, ctx.LifeBrush); // life + DrawSampleLine(ctx, 2394, 1185, 1411, ctx.ManaBrush); // mana + } } private static void DrawBar(D2dRenderContext ctx, float x, float y, float pct, - ID2D1SolidColorBrush fillBrush, Screen.HudValues? values, + ID2D1SolidColorBrush fillBrush, ref string? cachedLabel, ref IDWriteTextLayout? cachedLayout) { var rt = ctx.RenderTarget; @@ -42,31 +57,57 @@ internal sealed class D2dHudInfoLayer : ID2dOverlayLayer, IDisposable rt.DrawRectangle(outer, ctx.Gray, 1f); // Fill - var fillWidth = BarWidth * Math.Clamp(pct, 0, 1); + var clamped = Math.Clamp(pct, 0, 1); + var fillWidth = BarWidth * clamped; if (fillWidth > 0) rt.FillRectangle(new RectangleF(x, y, fillWidth, BarHeight), fillBrush); - // Value text - if (values != null) + // Percentage text + var label = $"{clamped:P0}"; + if (label != cachedLabel) { - var label = $"{values.Current}/{values.Max}"; - if (label != cachedLabel) - { - cachedLayout?.Dispose(); - cachedLabel = label; - cachedLayout = ctx.CreateTextLayout(label, ctx.BarValueFormat); - } - - var m = cachedLayout!.Metrics; - var textX = x + (BarWidth - m.Width) / 2; - var textY = y + (BarHeight - m.Height) / 2; - rt.DrawTextLayout(new System.Numerics.Vector2(textX, textY), cachedLayout, ctx.White); + cachedLayout?.Dispose(); + cachedLabel = label; + cachedLayout = ctx.CreateTextLayout(label, ctx.BarValueFormat); } + + var m = cachedLayout!.Metrics; + var textX = x + (BarWidth - m.Width) / 2; + var textY = y + (BarHeight - m.Height) / 2; + rt.DrawTextLayout(new System.Numerics.Vector2(textX, textY), cachedLayout, ctx.White); + } + + private static void DrawShieldArc(D2dRenderContext ctx) + { + const float cx = 170, cy = 1298, r = 130; + var rt = ctx.RenderTarget; + + // Draw dots along the right semicircle (-90° to +90°) + for (int deg = -90; deg <= 90; deg += 2) + { + var rad = deg * Math.PI / 180.0; + var x = (float)(cx + r * Math.Cos(rad)); + var y = (float)(cy + r * Math.Sin(rad)); + rt.FillRectangle(new RectangleF(x - 1, y - 1, 3, 3), ctx.Yellow); + } + + // Draw center cross + rt.FillRectangle(new RectangleF(cx - 3, cy - 1, 7, 3), ctx.Yellow); + rt.FillRectangle(new RectangleF(cx - 1, cy - 3, 3, 7), ctx.Yellow); + } + + private static void DrawSampleLine(D2dRenderContext ctx, float x, float yTop, float yBot, ID2D1SolidColorBrush brush) + { + ctx.RenderTarget.DrawLine( + new System.Numerics.Vector2(x, yTop), + new System.Numerics.Vector2(x, yBot), + brush, 2f); } public void Dispose() { _lifeLayout?.Dispose(); + _shieldLayout?.Dispose(); _manaLayout?.Dispose(); } } diff --git a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs index 5d04872..b403995 100644 --- a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs @@ -12,6 +12,11 @@ public partial class DebugViewModel : ObservableObject [ObservableProperty] private string _findText = ""; [ObservableProperty] private string _debugResult = ""; + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(BurstCaptureLabel))] + private bool _isBurstCapturing; + + public string BurstCaptureLabel => IsBurstCapturing ? "Stop Capture" : "Burst Capture"; [ObservableProperty] private string _selectedGridLayout = "inventory"; [ObservableProperty] private decimal? _clickX; [ObservableProperty] private decimal? _clickY; @@ -148,6 +153,15 @@ public partial class DebugViewModel : ObservableObject } } + [RelayCommand] + private void DetectionStatus() + { + var enemy = _bot.EnemyDetector.Latest; + var boss = _bot.BossDetector.Latest; + DebugResult = $"Enemy: enabled={_bot.EnemyDetector.Enabled}, count={enemy.Enemies.Count}, ms={enemy.InferenceMs:F1}\n" + + $"Boss: enabled={_bot.BossDetector.Enabled}, count={boss.Bosses.Count}, ms={boss.InferenceMs:F1}"; + } + [RelayCommand] private void SaveMinimapDebug() { @@ -187,6 +201,116 @@ public partial class DebugViewModel : ObservableObject catch (Exception ex) { DebugResult = $"Failed: {ex.Message}"; } } + [RelayCommand] + private async Task AttackTest() + { + const int VK_Q = 0x51; + const int DurationMs = 30_000; + const int PollMs = 100; + const float ManaLow = 0.50f; + const float ManaResume = 0.75f; + const float ManaQThreshold = 0.60f; + const int QPhaseStableMs = 2_000; + const int QCooldownMs = 5_000; + + var rng = new Random(); + try + { + DebugResult = "Attack test: focusing game..."; + await _bot.Game.FocusGame(); + await _bot.Game.MoveMouseTo(1280, 720); + await Task.Delay(300); + + var holding = true; + _bot.Game.LeftMouseDown(); + _bot.Game.RightMouseDown(); + + var sw = System.Diagnostics.Stopwatch.StartNew(); + var manaStableStart = (long?)null; + var qPhase = false; + long lastQTime = -QCooldownMs; + + while (sw.ElapsedMilliseconds < DurationMs) + { + var mana = _bot.HudReader.Current.ManaPct; + var elapsed = sw.ElapsedMilliseconds; + + // Mana management + if (holding && mana < ManaLow) + { + _bot.Game.LeftMouseUp(); + _bot.Game.RightMouseUp(); + holding = false; + DebugResult = $"Attack test: mana low ({mana:P0}), waiting..."; + await Task.Delay(50 + rng.Next(100)); + } + else if (!holding && mana >= ManaResume) + { + await Task.Delay(50 + rng.Next(100)); + _bot.Game.LeftMouseDown(); + _bot.Game.RightMouseDown(); + holding = true; + DebugResult = $"Attack test: mana recovered ({mana:P0}), attacking..."; + } + + // Track Q phase activation + if (!qPhase) + { + if (mana > ManaQThreshold) + { + manaStableStart ??= elapsed; + if (elapsed - manaStableStart.Value >= QPhaseStableMs) + { + qPhase = true; + DebugResult = "Attack test: Q phase activated"; + } + } + else + { + manaStableStart = null; + } + } + + // Press Q+E periodically + if (qPhase && holding && elapsed - lastQTime >= QCooldownMs) + { + await _bot.Game.PressKey(VK_Q); + await Task.Delay(100 + rng.Next(100)); + _bot.Game.LeftMouseUp(); + _bot.Game.RightMouseUp(); + await Task.Delay(200 + rng.Next(100)); + _bot.Game.LeftMouseDown(); + _bot.Game.RightMouseDown(); + lastQTime = elapsed; + } + + await Task.Delay(PollMs + rng.Next(100)); + } + + DebugResult = "Attack test: completed (30s)"; + } + catch (Exception ex) + { + DebugResult = $"Attack test failed: {ex.Message}"; + Log.Error(ex, "Attack test failed"); + } + finally + { + _bot.Game.LeftMouseUp(); + _bot.Game.RightMouseUp(); + } + } + + [RelayCommand] + private void ToggleBurstCapture() + { + IsBurstCapturing = !IsBurstCapturing; + _bot.FrameSaver.BurstMode = IsBurstCapturing; + DebugResult = IsBurstCapturing + ? "Burst capture ON — saving every 200ms to training-data/raw/" + : $"Burst capture OFF — {_bot.FrameSaver.SavedCount} frames saved"; + } + [RelayCommand] private async Task ClickSalvage() { diff --git a/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs b/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs index 32d1e69..edd75f9 100644 --- a/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/MainWindowViewModel.cs @@ -194,11 +194,12 @@ public partial class MainWindowViewModel : ObservableObject { Log.Information("END pressed — emergency stop"); await _bot.Navigation.Stop(); + _bot.BossRunExecutor.Stop(); _bot.Pause(); Avalonia.Threading.Dispatcher.UIThread.Post(() => { IsPaused = true; - State = "Stopped (F12)"; + State = "Stopped (END)"; }); } f12WasDown = endDown; diff --git a/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs b/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs index 9104b4c..2b83730 100644 --- a/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs @@ -1,3 +1,4 @@ +using System.Collections.ObjectModel; using Timer = System.Timers.Timer; using Avalonia.Threading; using CommunityToolkit.Mvvm.ComponentModel; @@ -19,16 +20,33 @@ public partial class MappingViewModel : ObservableObject, IDisposable [ObservableProperty] private int _enemiesDetected; [ObservableProperty] private float _inferenceMs; [ObservableProperty] private bool _hasModel; + [ObservableProperty] private bool _isKulemak; + [ObservableProperty] private bool _kulemakEnabled; + [ObservableProperty] private string _invitationTabPath = ""; + [ObservableProperty] private string _lootTabPath = ""; + [ObservableProperty] private decimal? _invitationCount = 15; - public static MapType[] MapTypes { get; } = [MapType.TrialOfChaos, MapType.Temple, MapType.Endgame]; + public static MapType[] MapTypes { get; } = [MapType.TrialOfChaos, MapType.Temple, MapType.Endgame, MapType.Kulemak]; + public ObservableCollection StashTabPaths { get; } = []; - private static readonly string ModelPath = Path.GetFullPath("tools/python-detect/models/enemy-v1.pt"); + private static readonly string ModelsDir = Path.GetFullPath("tools/python-detect/models"); + + private static bool AnyModelExists() => + Directory.Exists(ModelsDir) && Directory.GetFiles(ModelsDir, "*.pt").Length > 0; public MappingViewModel(BotOrchestrator bot) { _bot = bot; _selectedMapType = bot.Config.MapType; - _hasModel = File.Exists(ModelPath); + _isKulemak = _selectedMapType == MapType.Kulemak; + _hasModel = AnyModelExists(); + + // Load Kulemak settings + _kulemakEnabled = bot.Config.Kulemak.Enabled; + _invitationTabPath = bot.Config.Kulemak.InvitationTabPath; + _lootTabPath = bot.Config.Kulemak.LootTabPath; + _invitationCount = bot.Config.Kulemak.InvitationCount; + LoadStashTabPaths(); _bot.EnemyDetector.DetectionUpdated += OnDetectionUpdated; @@ -40,6 +58,47 @@ public partial class MappingViewModel : ObservableObject, IDisposable partial void OnSelectedMapTypeChanged(MapType value) { _bot.Store.UpdateSettings(s => s.MapType = value); + IsKulemak = value == MapType.Kulemak; + } + + partial void OnKulemakEnabledChanged(bool value) + { + _bot.Store.UpdateSettings(s => s.Kulemak.Enabled = value); + } + + partial void OnInvitationTabPathChanged(string value) + { + _bot.Store.UpdateSettings(s => s.Kulemak.InvitationTabPath = value); + } + + partial void OnLootTabPathChanged(string value) + { + _bot.Store.UpdateSettings(s => s.Kulemak.LootTabPath = value); + } + + partial void OnInvitationCountChanged(decimal? value) + { + _bot.Store.UpdateSettings(s => s.Kulemak.InvitationCount = (int)(value ?? 15)); + } + + private void LoadStashTabPaths() + { + StashTabPaths.Clear(); + StashTabPaths.Add(""); // empty = not configured + var s = _bot.Store.Settings; + if (s.StashCalibration == null) return; + foreach (var tab in s.StashCalibration.Tabs) + { + if (tab.IsFolder) + { + foreach (var sub in tab.SubTabs) + StashTabPaths.Add($"{tab.Name}/{sub.Name}"); + } + else + { + StashTabPaths.Add(tab.Name); + } + } } partial void OnIsFrameSaverEnabledChanged(bool value) @@ -50,6 +109,7 @@ public partial class MappingViewModel : ObservableObject, IDisposable partial void OnIsDetectionEnabledChanged(bool value) { _bot.EnemyDetector.Enabled = value; + _bot.BossDetector.Enabled = value; } private void OnDetectionUpdated(DetectionSnapshot snapshot) @@ -64,7 +124,7 @@ public partial class MappingViewModel : ObservableObject, IDisposable private void RefreshStats() { FramesSaved = _bot.FrameSaver.SavedCount; - HasModel = File.Exists(ModelPath); + HasModel = AnyModelExists(); } public void Dispose() diff --git a/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs b/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs index e0d5f37..2aa3f55 100644 --- a/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs @@ -19,6 +19,7 @@ public partial class SettingsViewModel : ObservableObject [ObservableProperty] private decimal? _waitForMoreItemsMs = 20000; [ObservableProperty] private decimal? _betweenTradesDelayMs = 5000; [ObservableProperty] private bool _headless = true; + [ObservableProperty] private bool _showHudDebug; [ObservableProperty] private bool _isSaved; [ObservableProperty] private string _calibrationStatus = ""; [ObservableProperty] private string _stashCalibratedAt = ""; @@ -44,6 +45,7 @@ public partial class SettingsViewModel : ObservableObject WaitForMoreItemsMs = s.WaitForMoreItemsMs; BetweenTradesDelayMs = s.BetweenTradesDelayMs; Headless = s.Headless; + ShowHudDebug = s.ShowHudDebug; } private void LoadTabs() @@ -94,6 +96,7 @@ public partial class SettingsViewModel : ObservableObject s.WaitForMoreItemsMs = (int)(WaitForMoreItemsMs ?? 20000); s.BetweenTradesDelayMs = (int)(BetweenTradesDelayMs ?? 5000); s.Headless = Headless; + s.ShowHudDebug = ShowHudDebug; }); IsSaved = true; @@ -206,4 +209,5 @@ public partial class SettingsViewModel : ObservableObject partial void OnWaitForMoreItemsMsChanged(decimal? value) => IsSaved = false; partial void OnBetweenTradesDelayMsChanged(decimal? value) => IsSaved = false; partial void OnHeadlessChanged(bool value) => IsSaved = false; + partial void OnShowHudDebugChanged(bool value) => IsSaved = false; } diff --git a/src/Poe2Trade.Ui/Views/MainWindow.axaml b/src/Poe2Trade.Ui/Views/MainWindow.axaml index 4aca908..3ea8a1e 100644 --- a/src/Poe2Trade.Ui/Views/MainWindow.axaml +++ b/src/Poe2Trade.Ui/Views/MainWindow.axaml @@ -233,6 +233,36 @@ + + + + + + + + + + + + + + + + + + + + @@ -298,6 +328,10 @@