diff --git a/debug_loot_capture.png b/debug_loot_capture.png index abea196..a5cd149 100644 Binary files a/debug_loot_capture.png and b/debug_loot_capture.png differ diff --git a/debug_loot_detected.png b/debug_loot_detected.png index 68e75b2..85ba59a 100644 Binary files a/debug_loot_detected.png and b/debug_loot_detected.png differ diff --git a/debug_loot_edges.png b/debug_loot_edges.png index 07b4b1f..41fc8a0 100644 Binary files a/debug_loot_edges.png and b/debug_loot_edges.png differ diff --git a/src/Poe2Trade.Bot/BossRunExecutor.cs b/src/Poe2Trade.Bot/BossRunExecutor.cs index c706344..5e2f538 100644 --- a/src/Poe2Trade.Bot/BossRunExecutor.cs +++ b/src/Poe2Trade.Bot/BossRunExecutor.cs @@ -107,6 +107,7 @@ public class BossRunExecutor : GameExecutor if (_stopped) break; SetState(BossRunState.Looting); + await Helpers.Sleep(1000); // wait for loot labels to render await Loot(); if (_stopped) break; @@ -316,7 +317,7 @@ public class BossRunExecutor : GameExecutor Log.Information("Fight phase starting"); // Wait for arena to settle - await Helpers.Sleep(4500); + await Helpers.Sleep(3000); if (_stopped) return; // Find and click the cathedral door @@ -343,7 +344,8 @@ public class BossRunExecutor : GameExecutor const double wellWorldY = -378; await WalkToWorldPosition(fightWorldX, fightWorldY, cancelWhen: IsBossAlive); - if (_stopped) return; + _nav.Frozen = true; // Lock canvas — position tracking only + if (_stopped) { _nav.Frozen = false; return; } // 3x fight-then-well loop for (var phase = 1; phase <= 3; phase++) @@ -351,7 +353,7 @@ public class BossRunExecutor : GameExecutor if (_stopped) return; Log.Information("=== Boss phase {Phase}/4 ===", phase); - var lastBossPos = await AttackBossUntilGone(); + var lastBossPos = await AttackBossUntilGone(fightWorldX, fightWorldY); if (_stopped) return; // Update fight area to where the boss was last seen @@ -368,7 +370,7 @@ public class BossRunExecutor : GameExecutor // Walk to well and click the closest match to screen center Log.Information("Phase {Phase} done, walking to well", phase); await WalkToWorldPosition(wellWorldX, wellWorldY); - await Helpers.Sleep(1000); + await Helpers.Sleep(500); await ClickClosestTemplateToCenter(CathedralWellTemplate); await Helpers.Sleep(200); @@ -379,29 +381,32 @@ public class BossRunExecutor : GameExecutor // 4th fight - no well after if (_stopped) return; Log.Information("=== Boss phase 4/4 ==="); - var phase4BossPos = await AttackBossUntilGone(); + var finalBossPos = await AttackBossUntilGone(fightWorldX, fightWorldY); if (_stopped) return; - // Walk toward where the boss died (ring spawns there) - var ringX = phase4BossPos?.X ?? fightWorldX; - var ringY = phase4BossPos?.Y ?? fightWorldY; - Log.Information("Walking to ring area ({X:F0},{Y:F0})", ringX, ringY); - await WalkToWorldPosition(ringX, ringY); + // Update fight area from phase 4 if we got detections + if (finalBossPos != null) + { + fightWorldX = finalBossPos.Value.X; + fightWorldY = finalBossPos.Value.Y; + } + + // Walk to known ring position and look for the template + await WalkToWorldPosition(-440, -330); + await Helpers.Sleep(1000); if (_stopped) return; Log.Information("Looking for Return the Ring..."); var ring = await _screen.TemplateMatch(ReturnTheRingTemplate); if (ring == null) { - Log.Warning("Could not find Return the Ring template, retrying after 2s..."); - await Helpers.Sleep(2000); ring = await _screen.TemplateMatch(ReturnTheRingTemplate); } if (ring != null) { Log.Information("Found Return the Ring at ({X},{Y}), clicking", ring.X, ring.Y); await _game.LeftClickAt(ring.X, ring.Y); - await Helpers.Sleep(2000); + await Helpers.Sleep(500); } else { @@ -409,27 +414,20 @@ public class BossRunExecutor : GameExecutor } if (_stopped) return; - // Walk up and press Q - Log.Information("Walking up and pressing Q"); - await _game.KeyDown(InputSender.VK.W); - await Helpers.Sleep(1500); - await _game.KeyUp(InputSender.VK.W); + // Walk back to fight area — fightWorldX/Y carries position from all phases + Log.Information("Walking to fight position ({X:F0},{Y:F0})", fightWorldX, fightWorldY); + await WalkToWorldPosition(fightWorldX, fightWorldY); await Helpers.Sleep(300); await _game.PressKey(InputSender.VK.Q); - await Helpers.Sleep(500); - - // Spam L+R at position for 7s - Log.Information("Attacking at ring fight position (Q phase)"); - await AttackAtPosition(1280, 720, 7000); + await Helpers.Sleep(300); + await _game.PressKey(InputSender.VK.E); + await Helpers.Sleep(300); + Log.Information("Attacking at ring fight position"); + await AttackBossUntilGone(fightWorldX, fightWorldY); if (_stopped) return; - // Press E, spam L+R at same position for 7s - Log.Information("Pressing E and continuing attack"); - await _game.PressKey(InputSender.VK.E); - await Helpers.Sleep(500); - await AttackAtPosition(1280, 720, 7000); - StopBossDetection(); + _nav.Frozen = false; Log.Information("Fight complete"); } @@ -468,7 +466,8 @@ public class BossRunExecutor : GameExecutor if (now - _lastDeathCheckMs < 2000) return false; _lastDeathCheckMs = now; - var match = await _screen.TemplateMatch(ResurrectTemplate); + var deathRegion = new Region(1090, 1030, 370, 60); + var match = await _screen.TemplateMatch(ResurrectTemplate, deathRegion); if (match == null) return false; Log.Warning("Death detected! Clicking resurrect at ({X},{Y})", match.X, match.Y); @@ -522,6 +521,27 @@ public class BossRunExecutor : GameExecutor try { await combatTask; } catch { /* expected */ } await _combat.ReleaseAll(); cts.Dispose(); + await WaitForStablePosition(); + } + + private async Task WaitForStablePosition(int minConsecutive = 5, int timeoutMs = 2000) + { + var sw = Stopwatch.StartNew(); + var consecutive = 0; + while (sw.ElapsedMilliseconds < timeoutMs && consecutive < minConsecutive) + { + await Helpers.Sleep(50); + if (_nav.LastMatchSucceeded) + consecutive++; + else + consecutive = 0; + } + if (consecutive >= minConsecutive) + Log.Information("Position stabilized ({Count} consecutive matches in {Ms}ms)", + consecutive, sw.ElapsedMilliseconds); + else + Log.Warning("Position stabilization timed out ({Count}/{Min} after {Ms}ms)", + consecutive, minConsecutive, sw.ElapsedMilliseconds); } private async Task WaitForBossSpawn(int timeoutMs = 30_000) @@ -554,6 +574,14 @@ public class BossRunExecutor : GameExecutor } } + // Final death check before giving up — force check ignoring throttle + _lastDeathCheckMs = 0; + if (await CheckDeath()) + { + Log.Warning("Died while waiting for boss spawn"); + return false; + } + Log.Warning("WaitForBossSpawn timed out after {Ms}ms", timeoutMs); return false; } @@ -567,7 +595,8 @@ public class BossRunExecutor : GameExecutor /// Wait for boss to spawn, then attack until healthbar disappears. /// Returns the last world position where YOLO spotted the boss, or null. /// - private async Task<(double X, double Y)?> AttackBossUntilGone(int timeoutMs = 120_000) + private async Task<(double X, double Y)?> AttackBossUntilGone( + double fightAreaX = double.NaN, double fightAreaY = double.NaN, int timeoutMs = 120_000) { // Wait for boss to actually appear before attacking if (!await WaitForBossSpawn()) @@ -575,6 +604,7 @@ public class BossRunExecutor : GameExecutor const int screenCx = 1280; const int screenCy = 720; + const double screenToWorld = 97.0 / 835.0; (double X, double Y)? lastBossWorldPos = null; Log.Information("Boss is alive, engaging"); @@ -583,6 +613,9 @@ public class BossRunExecutor : GameExecutor try { var sw = Stopwatch.StartNew(); + var healthbarMissCount = 0; + const int healthbarMissThreshold = 3; // require 3 consecutive misses + while (sw.ElapsedMilliseconds < timeoutMs) { if (_stopped) return lastBossWorldPos; @@ -595,19 +628,55 @@ public class BossRunExecutor : GameExecutor _combatTargetX = boss.Cx; _combatTargetY = boss.Cy; - const double screenToWorld = 97.0 / 835.0; var wp = _nav.WorldPosition; lastBossWorldPos = ( wp.X + (boss.Cx - screenCx) * screenToWorld, wp.Y + (boss.Cy - screenCy) * screenToWorld); + + // Walk toward boss to stay as close as possible + var bossDx = boss.Cx - screenCx; + var bossDy = boss.Cy - screenCy; + var bossDist = Math.Sqrt(bossDx * bossDx + bossDy * bossDy); + + if (bossDist > 50) + { + var dirX = bossDx / bossDist; + var dirY = bossDy / bossDist; + + var keys = new List(); + if (dirY < -0.3) keys.Add(InputSender.VK.W); + if (dirY > 0.3) keys.Add(InputSender.VK.S); + if (dirX < -0.3) keys.Add(InputSender.VK.A); + if (dirX > 0.3) keys.Add(InputSender.VK.D); + + foreach (var k in keys) await _game.KeyDown(k); + await Helpers.Sleep(150); + foreach (var k in keys) await _game.KeyUp(k); + } } // Check death + healthbar (combat keeps running in background) - if (await CheckDeath()) continue; + if (await CheckDeath()) { healthbarMissCount = 0; continue; } - if (!await IsBossAlive()) + if (await IsBossAlive()) { - Log.Information("Healthbar not found, boss phase over after {Ms}ms", sw.ElapsedMilliseconds); + healthbarMissCount = 0; + } + else + { + healthbarMissCount++; + if (healthbarMissCount < healthbarMissThreshold) + { + Log.Debug("Healthbar miss {N}/{Threshold}", healthbarMissCount, healthbarMissThreshold); + continue; + } + + // Confirm: did we die or did boss phase actually end? + _lastDeathCheckMs = 0; + if (await CheckDeath()) { healthbarMissCount = 0; continue; } + + Log.Information("Healthbar gone for {N} checks, boss phase over after {Ms}ms", + healthbarMissCount, sw.ElapsedMilliseconds); return lastBossWorldPos; } } @@ -640,6 +709,33 @@ public class BossRunExecutor : GameExecutor } } + private async Task AttackUntilBossDead(int x, int y, int timeoutMs) + { + var (combatTask, cts) = StartCombatLoop(x, y, jitter: 20); + try + { + var sw = Stopwatch.StartNew(); + // Attack for at least 2s before checking healthbar + await Helpers.Sleep(2000); + while (sw.ElapsedMilliseconds < timeoutMs) + { + if (_stopped) return; + if (await CheckDeath()) continue; + if (!await IsBossAlive()) + { + Log.Information("Boss dead, stopping attack after {Ms}ms", sw.ElapsedMilliseconds); + return; + } + await Helpers.Sleep(500); + } + Log.Warning("AttackUntilBossDead timed out after {Ms}ms", timeoutMs); + } + finally + { + await StopCombatLoop(combatTask, cts); + } + } + /// /// Find all template matches and click the one closest to screen center. /// @@ -648,24 +744,38 @@ public class BossRunExecutor : GameExecutor const int screenCx = 2560 / 2; const int screenCy = 1440 / 2; - var matches = await _screen.TemplateMatchAll(templatePath); - if (matches.Count == 0) + // Search center region only to avoid clicking distant matches + var centerRegion = new Region(850, 50, 860, 550); + + for (var attempt = 0; attempt < 3; attempt++) { - Log.Warning("No matches found for {Template}, clicking screen center", Path.GetFileName(templatePath)); - await _game.LeftClickAt(screenCx, screenCy); - return; + var matches = await _screen.TemplateMatchAll(templatePath, centerRegion); + if (matches.Count > 0) + { + var closest = matches.OrderBy(m => + { + var dx = m.X - screenCx; + var dy = m.Y - screenCy; + return dx * dx + dy * dy; + }).First(); + + Log.Information("Clicking closest match at ({X},{Y}) conf={Conf:F3} (of {Count} matches, attempt {A})", + closest.X, closest.Y, closest.Confidence, matches.Count, attempt + 1); + await _game.LeftClickAt(closest.X, closest.Y); + return; + } + + // Nudge character a bit and retry + Log.Warning("No center match for {Template} (attempt {A}/3), nudging", Path.GetFileName(templatePath), attempt + 1); + var nudgeKey = attempt % 2 == 0 ? InputSender.VK.W : InputSender.VK.S; + await _game.KeyDown(nudgeKey); + await Helpers.Sleep(300); + await _game.KeyUp(nudgeKey); + await Helpers.Sleep(500); } - var closest = matches.OrderBy(m => - { - var dx = m.X - screenCx; - var dy = m.Y - screenCy; - return dx * dx + dy * dy; - }).First(); - - Log.Information("Clicking closest match at ({X},{Y}) conf={Conf:F3} (of {Count} matches)", - closest.X, closest.Y, closest.Confidence, matches.Count); - await _game.LeftClickAt(closest.X, closest.Y); + Log.Warning("No matches found for {Template} after nudges, clicking screen center", Path.GetFileName(templatePath)); + await _game.LeftClickAt(screenCx, screenCy); } /// @@ -772,17 +882,10 @@ public class BossRunExecutor : GameExecutor await _game.KeyUp(InputSender.VK.S); await Helpers.Sleep(200); - // Press + to open portal + // Press + to open portal, then click it at known position await _game.PressPlus(); - await Helpers.Sleep(800); - - // Find "The Ardura Caravan" and click it - var caravanPos = await _inventory.FindAndClickNameplate("The Ardura Caravan", maxRetries: 5, retryDelayMs: 1500); - if (caravanPos == null) - { - Log.Error("Could not find 'The Ardura Caravan' portal"); - return false; - } + await Helpers.Sleep(2500); + await _game.LeftClickAt(1280, 450); // Wait for area transition to caravan var arrivedCaravan = await _inventory.WaitForAreaTransition(_config.TravelTimeoutMs); @@ -816,6 +919,9 @@ public class BossRunExecutor : GameExecutor await _game.FocusGame(); await Helpers.Sleep(Delays.PostFocus); + // Identify items at Doryani before stashing + await _inventory.IdentifyItems(); + // Open stash var stashPos = await _inventory.FindAndClickNameplate("Stash"); if (stashPos == null) @@ -830,18 +936,26 @@ public class BossRunExecutor : GameExecutor if (lootTab != null) await _inventory.ClickStashTab(lootTab, lootFolder); - var scanResult = await _screen.Grid.Scan("inventory"); - if (scanResult.Occupied.Count > 0) + for (var pass = 0; pass < 3; pass++) { - Log.Information("Depositing {Count} items to loot tab", scanResult.Occupied.Count); + var scanResult = await _screen.Grid.Scan("inventory"); + if (scanResult.Items.Count == 0) + { + if (pass > 0) Log.Information("Inventory clear after {Pass} passes", pass); + break; + } + + Log.Information("Depositing {Count} items to loot tab (pass {Pass})", scanResult.Items.Count, pass + 1); await _game.KeyDown(InputSender.VK.SHIFT); await _game.HoldCtrl(); - foreach (var cell in scanResult.Occupied) + + foreach (var item in scanResult.Items) { - var center = _screen.Grid.GetCellCenter(GridLayouts.Inventory, cell.Row, cell.Col); + var center = _screen.Grid.GetCellCenter(GridLayouts.Inventory, item.Row, item.Col); await _game.LeftClickAt(center.X, center.Y); await Helpers.Sleep(Delays.ClickInterval); } + await _game.ReleaseCtrl(); await _game.KeyUp(InputSender.VK.SHIFT); await Helpers.Sleep(Delays.PostEscape); diff --git a/src/Poe2Trade.Bot/CombatManager.cs b/src/Poe2Trade.Bot/CombatManager.cs index a37d2e7..f085215 100644 --- a/src/Poe2Trade.Bot/CombatManager.cs +++ b/src/Poe2Trade.Bot/CombatManager.cs @@ -18,7 +18,8 @@ public class CombatManager // Orbit: cycle W→D→S→A to dodge in a small circle private static readonly int[] OrbitKeys = [InputSender.VK.W, InputSender.VK.D, InputSender.VK.S, InputSender.VK.A]; - private const int OrbitStepMs = 400; // time per direction + private const int OrbitStepMinMs = 60; // short tap per direction → ~10px radius + private const int OrbitStepMaxMs = 120; private readonly IGameController _game; private readonly HudReader _hudReader; @@ -29,6 +30,12 @@ public class CombatManager private readonly Stopwatch _orbitSw = Stopwatch.StartNew(); private int _orbitIndex = -1; private long _lastOrbitMs; + private int _nextOrbitMs = OrbitStepMinMs; + + // Smoothed mouse position — lerps toward target to avoid jitter + private double _smoothX = 1280; + private double _smoothY = 720; + private const double SmoothFactor = 0.25; // 0=no movement, 1=instant snap public bool IsHolding => _holding; @@ -47,6 +54,13 @@ public class CombatManager await _flasks.Tick(); await UpdateOrbit(); + // Lerp smoothed position toward target + _smoothX += (x - _smoothX) * SmoothFactor; + _smoothY += (y - _smoothY) * SmoothFactor; + + var mouseX = (int)_smoothX + Rng.Next(-jitter, jitter + 1); + var mouseY = (int)_smoothY + Rng.Next(-jitter, jitter + 1); + var mana = _hudReader.Current.ManaPct; if (!_holding) @@ -56,9 +70,7 @@ public class CombatManager else _manaStableCount = 0; - var targetX = x + Rng.Next(-jitter, jitter + 1); - var targetY = y + Rng.Next(-jitter, jitter + 1); - await _game.MoveMouseFast(targetX, targetY); + await _game.MoveMouseFast(mouseX, mouseY); _game.LeftMouseDown(); await Helpers.Sleep(Rng.Next(20, 35)); @@ -79,9 +91,7 @@ public class CombatManager } else { - var targetX = x + Rng.Next(-jitter, jitter + 1); - var targetY = y + Rng.Next(-jitter, jitter + 1); - await _game.MoveMouseFast(targetX, targetY); + await _game.MoveMouseFast(mouseX, mouseY); if (mana < 0.30f) { @@ -102,15 +112,17 @@ public class CombatManager private async Task UpdateOrbit() { var now = _orbitSw.ElapsedMilliseconds; - if (now - _lastOrbitMs < OrbitStepMs) return; + if (now - _lastOrbitMs < _nextOrbitMs) return; _lastOrbitMs = now; + _nextOrbitMs = Rng.Next(OrbitStepMinMs, OrbitStepMaxMs + 1); // Release previous direction if (_orbitIndex >= 0) await _game.KeyUp(OrbitKeys[_orbitIndex]); - // Advance to next direction - _orbitIndex = (_orbitIndex + 1) % OrbitKeys.Length; + // Occasionally skip a direction to make movement less predictable + var skip = Rng.Next(0, 5) == 0 ? 2 : 1; + _orbitIndex = (_orbitIndex + skip) % OrbitKeys.Length; await _game.KeyDown(OrbitKeys[_orbitIndex]); } @@ -135,6 +147,8 @@ public class CombatManager } _holding = false; _manaStableCount = 0; + _smoothX = 1280; + _smoothY = 720; await ReleaseOrbit(); } diff --git a/src/Poe2Trade.Core/Helpers.cs b/src/Poe2Trade.Core/Helpers.cs index 6442c0f..b6afcc8 100644 --- a/src/Poe2Trade.Core/Helpers.cs +++ b/src/Poe2Trade.Core/Helpers.cs @@ -4,7 +4,12 @@ public static class Helpers { private static readonly Random Rng = new(); - public static Task Sleep(int ms) => Task.Delay(ms); + public static Task Sleep(int ms) + { + var variance = Math.Max(1, ms / 10); // ±10% + var actual = ms + Rng.Next(-variance, variance + 1); + return Task.Delay(Math.Max(1, actual)); + } public static Task RandomDelay(int minMs, int maxMs) { diff --git a/src/Poe2Trade.Game/InputSender.cs b/src/Poe2Trade.Game/InputSender.cs index 7c93bc2..b984d24 100644 --- a/src/Poe2Trade.Game/InputSender.cs +++ b/src/Poe2Trade.Game/InputSender.cs @@ -119,14 +119,14 @@ public class InputSender var perpX = -dy / distance; var perpY = dx / distance; - var spread = distance * 0.3; + var spread = distance * 0.15; - var cp1X = sx + dx * 0.25 + perpX * (Rng.NextDouble() - 0.5) * spread; - var cp1Y = sy + dy * 0.25 + perpY * (Rng.NextDouble() - 0.5) * spread; - var cp2X = sx + dx * 0.75 + perpX * (Rng.NextDouble() - 0.5) * spread; - var cp2Y = sy + dy * 0.75 + perpY * (Rng.NextDouble() - 0.5) * spread; + var cp1X = sx + dx * 0.3 + perpX * (Rng.NextDouble() - 0.5) * spread; + var cp1Y = sy + dy * 0.3 + perpY * (Rng.NextDouble() - 0.5) * spread; + var cp2X = sx + dx * 0.7 + perpX * (Rng.NextDouble() - 0.5) * spread; + var cp2Y = sy + dy * 0.7 + perpY * (Rng.NextDouble() - 0.5) * spread; - var steps = Math.Clamp((int)Math.Round(distance / 30), 8, 20); + var steps = Math.Clamp((int)Math.Round(distance / 15), 12, 40); for (var i = 1; i <= steps; i++) { @@ -134,11 +134,8 @@ public class InputSender var t = EaseInOutQuad(rawT); var (px, py) = CubicBezier(t, sx, sy, cp1X, cp1Y, cp2X, cp2Y, x, y); - var jitterX = i < steps ? (int)Math.Round((Rng.NextDouble() - 0.5) * 2) : 0; - var jitterY = i < steps ? (int)Math.Round((Rng.NextDouble() - 0.5) * 2) : 0; - - MoveMouseRaw((int)Math.Round(px) + jitterX, (int)Math.Round(py) + jitterY); - await Task.Delay(1 + Rng.Next(2)); + MoveMouseRaw((int)Math.Round(px), (int)Math.Round(py)); + await Task.Delay(2 + Rng.Next(3)); } MoveMouseRaw(x, y); diff --git a/src/Poe2Trade.Inventory/IInventoryManager.cs b/src/Poe2Trade.Inventory/IInventoryManager.cs index cfdddc0..9a65ec3 100644 --- a/src/Poe2Trade.Inventory/IInventoryManager.cs +++ b/src/Poe2Trade.Inventory/IInventoryManager.cs @@ -15,9 +15,10 @@ public interface IInventoryManager Task EnsureAtOwnHideout(); Task ProcessInventory(); Task WaitForAreaTransition(int timeoutMs, Func? triggerAction = null); - Task<(int X, int Y)?> FindAndClickNameplate(string name, int maxRetries = 3, int retryDelayMs = 1000); + Task<(int X, int Y)?> FindAndClickNameplate(string name, int maxRetries = 3, int retryDelayMs = 1000, System.Drawing.Rectangle? scanRegion = null, string? savePath = null); Task DepositItemsToStash(List items); Task SalvageItems(List items); + Task IdentifyItems(); (bool[,] Grid, List Items, int Free) GetInventoryState(); Task ClickStashTab(StashTabInfo tab, StashTabInfo? parentFolder = null); void ResetStashTabState(); diff --git a/src/Poe2Trade.Inventory/InventoryManager.cs b/src/Poe2Trade.Inventory/InventoryManager.cs index 670af48..aa97111 100644 --- a/src/Poe2Trade.Inventory/InventoryManager.cs +++ b/src/Poe2Trade.Inventory/InventoryManager.cs @@ -10,6 +10,7 @@ public class InventoryManager : IInventoryManager { private static readonly string SalvageTemplate = Path.Combine("assets", "salvage.png"); + public event Action? Updated; public InventoryTracker Tracker { get; } = new(); @@ -155,6 +156,36 @@ public class InventoryManager : IInventoryManager return true; } + public async Task IdentifyItems() + { + var nameplate = await FindAndClickNameplate("Doryani"); + if (nameplate == null) + { + Log.Error("Could not find Doryani nameplate"); + return false; + } + await Helpers.Sleep(Delays.PostStashOpen); + + // Dialog appears below and to the right of the nameplate + var dialogRegion = new Region( + nameplate.Value.X, nameplate.Value.Y, + 460, 600); + var identifyPos = await _screen.FindTextInRegion(dialogRegion, "Identify"); + if (identifyPos.HasValue) + { + await _game.LeftClickAt(identifyPos.Value.X, identifyPos.Value.Y); + await Helpers.Sleep(Delays.PostEscape); + } + else + { + Log.Warning("'Identify Items' not found in dialog region"); + } + + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + return true; + } + private async Task CtrlClickItems(List items, GridLayout layout, int clickDelayMs = Delays.ClickInterval) { await _game.KeyDown(Game.InputSender.VK.SHIFT); @@ -207,7 +238,7 @@ public class InventoryManager : IInventoryManager } } - public async Task<(int X, int Y)?> FindAndClickNameplate(string name, int maxRetries = 3, int retryDelayMs = 1000) + public async Task<(int X, int Y)?> FindAndClickNameplate(string name, int maxRetries = 3, int retryDelayMs = 1000, System.Drawing.Rectangle? scanRegion = null, string? savePath = null) { for (var attempt = 1; attempt <= maxRetries; attempt++) { @@ -227,7 +258,10 @@ public class InventoryManager : IInventoryManager await _game.KeyUp(Game.InputSender.VK.MENU); // Diff OCR — only processes the bright nameplate regions - var result = await _screen.NameplateDiffOcr(reference, current); + var attemptSavePath = savePath != null + ? Path.Combine(Path.GetDirectoryName(savePath)!, $"{Path.GetFileNameWithoutExtension(savePath)}_attempt{attempt}{Path.GetExtension(savePath)}") + : null; + var result = await _screen.NameplateDiffOcr(reference, current, scanRegion, attemptSavePath); var pos = FindWordInOcrResult(result, name, fuzzy: true); if (pos.HasValue) { @@ -236,7 +270,7 @@ public class InventoryManager : IInventoryManager return pos; } - Log.Debug("Nameplate '{Name}' not found in diff OCR (attempt {Attempt}), text: {Text}", name, attempt, result.Text); + Log.Information("Nameplate '{Name}' not found in diff OCR (attempt {Attempt}), text: {Text}", name, attempt, result.Text); if (attempt < maxRetries) await Helpers.Sleep(retryDelayMs); } diff --git a/src/Poe2Trade.Navigation/NavigationExecutor.cs b/src/Poe2Trade.Navigation/NavigationExecutor.cs index 9bfbe2e..bf5a804 100644 --- a/src/Poe2Trade.Navigation/NavigationExecutor.cs +++ b/src/Poe2Trade.Navigation/NavigationExecutor.cs @@ -492,6 +492,8 @@ public class NavigationExecutor : IDisposable public bool IsExploring => _state != NavigationState.Idle && _state != NavigationState.Completed && _state != NavigationState.Failed; public MapPosition Position => _worldMap.Position; public MapPosition WorldPosition => _worldMap.WorldPosition; + public bool LastMatchSucceeded => _worldMap.LastMatchSucceeded; + public bool Frozen { get => _worldMap.Frozen; set => _worldMap.Frozen = value; } public byte[] GetMapSnapshot() => _worldMap.GetMapSnapshot(); public byte[] GetViewportSnapshot(int viewSize = 400) { diff --git a/src/Poe2Trade.Navigation/WorldMap.cs b/src/Poe2Trade.Navigation/WorldMap.cs index 79ed437..fa9ca41 100644 --- a/src/Poe2Trade.Navigation/WorldMap.cs +++ b/src/Poe2Trade.Navigation/WorldMap.cs @@ -35,6 +35,7 @@ public class WorldMap : IDisposable public MapPosition WorldPosition => new(_position.X - _worldOriginX, _position.Y - _worldOriginY); public bool LastMatchSucceeded { get; private set; } + public bool Frozen { get; set; } public int CanvasSize => _canvasSize; internal List? LastBfsPath => _pathFinder.LastResult?.Path; @@ -171,9 +172,12 @@ public class WorldMap : IDisposable var prevPos = _position; _position = matched; var stitchStart = sw.Elapsed.TotalMilliseconds; - StitchWithConfidence(classifiedMat, _position, boosted: false, mode: mode); - PaintExploredCircle(_position); - MergeCheckpoints(_position, classifiedMat.Width, checkpointsOff, checkpointsOn); + if (!Frozen) + { + StitchWithConfidence(classifiedMat, _position, boosted: false, mode: mode); + PaintExploredCircle(_position); + MergeCheckpoints(_position, classifiedMat.Width, checkpointsOff, checkpointsOn); + } var stitchMs = sw.Elapsed.TotalMilliseconds - stitchStart; var posDx = _position.X - prevPos.X; diff --git a/src/Poe2Trade.Screen/BossDetector.cs b/src/Poe2Trade.Screen/BossDetector.cs index 22f353b..94db95b 100644 --- a/src/Poe2Trade.Screen/BossDetector.cs +++ b/src/Poe2Trade.Screen/BossDetector.cs @@ -11,7 +11,8 @@ namespace Poe2Trade.Screen; /// public class BossDetector : IFrameConsumer, IDisposable { - private const int MinConsecutiveFrames = 2; + private const int MinConsecutiveFrames = 1; + private const int MinConsecutiveMisses = 5; // don't clear _latest on a single miss frame private const string ModelsDir = "tools/python-detect/models"; private OnnxYoloDetector? _detector; @@ -20,6 +21,7 @@ public class BossDetector : IFrameConsumer, IDisposable private volatile BossSnapshot _latest = new([], 0, 0); private BossSnapshot _previous = new([], 0, 0); private int _consecutiveDetections; + private int _consecutiveMisses; private int _inferenceCount; // Async frame-slot: Process() drops frame here, background loop runs YOLO @@ -137,6 +139,9 @@ public class BossDetector : IFrameConsumer, IDisposable old?.Dispose(); _frameReady.Reset(); _consecutiveDetections = 0; + _consecutiveMisses = 0; + _latest = new BossSnapshot([], 0, 0); + _previous = new BossSnapshot([], 0, 0); } private async Task InferenceLoop(CancellationToken ct) @@ -165,6 +170,7 @@ public class BossDetector : IFrameConsumer, IDisposable if (detections.Count > 0) { _consecutiveDetections++; + _consecutiveMisses = 0; var timestamp = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); var deltaMs = (float)(timestamp - _previous.Timestamp); @@ -197,8 +203,13 @@ public class BossDetector : IFrameConsumer, IDisposable } else { + _consecutiveMisses++; _consecutiveDetections = 0; - _latest = new BossSnapshot([], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), 0); + + // Only clear after several consecutive misses to avoid + // flickering when YOLO drops a frame intermittently + if (_consecutiveMisses >= MinConsecutiveMisses) + _latest = new BossSnapshot([], DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), 0); } } finally diff --git a/src/Poe2Trade.Screen/GridReader.cs b/src/Poe2Trade.Screen/GridReader.cs index 8e508a2..48518d4 100644 --- a/src/Poe2Trade.Screen/GridReader.cs +++ b/src/Poe2Trade.Screen/GridReader.cs @@ -113,13 +113,18 @@ public class GridReader }); } + private static readonly Random Rng = new(); + public (int X, int Y) GetCellCenter(GridLayout layout, int row, int col) { var cellW = (double)layout.Region.Width / layout.Cols; var cellH = (double)layout.Region.Height / layout.Rows; + // ±20% jitter within the cell so clicks aren't pixel-perfect + var jitterX = (int)(cellW * 0.2 * (Rng.NextDouble() * 2 - 1)); + var jitterY = (int)(cellH * 0.2 * (Rng.NextDouble() * 2 - 1)); return ( - (int)Math.Round(layout.Region.X + col * cellW + cellW / 2), - (int)Math.Round(layout.Region.Y + row * cellH + cellH / 2) + (int)Math.Round(layout.Region.X + col * cellW + cellW / 2) + jitterX, + (int)Math.Round(layout.Region.Y + row * cellH + cellH / 2) + jitterY ); } diff --git a/src/Poe2Trade.Screen/IScreenReader.cs b/src/Poe2Trade.Screen/IScreenReader.cs index a5e371a..d044f57 100644 --- a/src/Poe2Trade.Screen/IScreenReader.cs +++ b/src/Poe2Trade.Screen/IScreenReader.cs @@ -18,7 +18,7 @@ public interface IScreenReader : IDisposable Task DiffOcr(string? savePath = null, Region? region = null); Task TemplateMatch(string templatePath, Region? region = null); Task> TemplateMatchAll(string templatePath, Region? region = null, double threshold = 0.7, bool silent = false); - Task NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current); + Task NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current, System.Drawing.Rectangle? scanRegion = null, string? savePath = null); void SetLootBaseline(System.Drawing.Bitmap frame); List DetectLootLabels(System.Drawing.Bitmap reference, System.Drawing.Bitmap current); System.Drawing.Bitmap CaptureRawBitmap(); diff --git a/src/Poe2Trade.Screen/ScreenReader.cs b/src/Poe2Trade.Screen/ScreenReader.cs index 200542c..8eeb2d9 100644 --- a/src/Poe2Trade.Screen/ScreenReader.cs +++ b/src/Poe2Trade.Screen/ScreenReader.cs @@ -212,16 +212,28 @@ public class ScreenReader : IScreenReader // Nameplate search region — skip top HUD, bottom bar, and side margins private const int NpTop = 120, NpBottom = 1080, NpMargin = 300; - public Task NameplateDiffOcr(Bitmap reference, Bitmap current) + public Task NameplateDiffOcr(Bitmap reference, Bitmap current, Rectangle? scanRegion = null, string? savePath = null) { int w = Math.Min(reference.Width, current.Width); int h = Math.Min(reference.Height, current.Height); - // Clamp search region to image bounds - int scanY0 = Math.Min(NpTop, h); - int scanY1 = Math.Min(NpBottom, h); - int scanX0 = Math.Min(NpMargin, w); - int scanX1 = Math.Max(scanX0, w - NpMargin); + // Use provided scan region or fall back to default nameplate bounds + int scanY0, scanY1, scanX0, scanX1; + if (scanRegion.HasValue) + { + var r = scanRegion.Value; + scanY0 = Math.Clamp(r.Y, 0, h); + scanY1 = Math.Clamp(r.Y + r.Height, 0, h); + scanX0 = Math.Clamp(r.X, 0, w); + scanX1 = Math.Clamp(r.X + r.Width, 0, w); + } + else + { + scanY0 = Math.Min(NpTop, h); + scanY1 = Math.Min(NpBottom, h); + scanX0 = Math.Min(NpMargin, w); + scanX1 = Math.Max(scanX0, w - NpMargin); + } var refData = reference.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); @@ -238,6 +250,75 @@ public class ScreenReader : IScreenReader const int brightThresh = 30; int scanW = scanX1 - scanX0; int scanH = scanY1 - scanY0; + + // When a scan region is provided, just crop & diff-mask that region directly (no cluster stitching) + if (scanRegion.HasValue) + { + using var crop = new Bitmap(scanW, scanH, PixelFormat.Format32bppArgb); + var cropData = crop.LockBits(new Rectangle(0, 0, scanW, scanH), ImageLockMode.WriteOnly, PixelFormat.Format32bppArgb); + byte[] cropPx = new byte[cropData.Stride * scanH]; + int cropStride = cropData.Stride; + + // Copy current image pixels, zeroing out any that didn't get brighter (diff mask) + Parallel.For(0, scanH, sy => + { + int y = sy + scanY0; + int srcOff = y * stride; + int dstOff = sy * cropStride; + for (int sx = 0; sx < scanW; sx++) + { + int x = sx + scanX0; + int si = srcOff + x * 4; + int di = dstOff + sx * 4; + int brighter = (curPx[si] - refPx[si]) + (curPx[si + 1] - refPx[si + 1]) + (curPx[si + 2] - refPx[si + 2]); + if (brighter > brightThresh) + { + cropPx[di] = curPx[si]; + cropPx[di + 1] = curPx[si + 1]; + cropPx[di + 2] = curPx[si + 2]; + cropPx[di + 3] = 255; + } + // else stays black (zeroed) + } + }); + + Marshal.Copy(cropPx, 0, cropData.Scan0, cropPx.Length); + crop.UnlockBits(cropData); + + if (savePath != null) + { + try + { + var dir = Path.GetDirectoryName(savePath); + if (!string.IsNullOrEmpty(dir)) Directory.CreateDirectory(dir); + crop.Save(savePath, System.Drawing.Imaging.ImageFormat.Png); + Log.Information("NameplateDiffOcr: saved crop to {Path}", savePath); + } + catch (Exception ex) { Log.Warning(ex, "NameplateDiffOcr: failed to save crop"); } + } + + var ocrSw2 = System.Diagnostics.Stopwatch.StartNew(); + OcrResponse ocrResult2; + try { ocrResult2 = _pythonBridge.OcrFromBitmap(crop); } + catch (TimeoutException) + { + Log.Warning("NameplateDiffOcr: crop OCR timed out"); + return Task.FromResult(new OcrResponse { Text = "", Lines = [] }); + } + Log.Information("NameplateDiffOcr: crop OCR in {Ms}ms", ocrSw2.ElapsedMilliseconds); + + // Offset coordinates back to screen space + foreach (var line in ocrResult2.Lines) + foreach (var word in line.Words) + { + word.X += scanX0; + word.Y += scanY0; + } + + return Task.FromResult(ocrResult2); + } + + // Full-screen path: cluster detection + stitching bool[] mask = new bool[scanW * scanH]; Parallel.For(0, scanH, sy => { @@ -321,9 +402,9 @@ public class ScreenReader : IScreenReader foreach (var word in line.Words) { // Find which crop this word belongs to by Y position - var crop = crops.Last(c => word.Y >= c.stitchY); - word.X += crop.screenX; - word.Y = word.Y - crop.stitchY + crop.screenY; + var crop2 = crops.Last(c => word.Y >= c.stitchY); + word.X += crop2.screenX; + word.Y = word.Y - crop2.stitchY + crop2.screenY; } } diff --git a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs index 85e1a8c..8d4f5c1 100644 --- a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs @@ -1,6 +1,8 @@ using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using Poe2Trade.Bot; +using Poe2Trade.Core; +using Poe2Trade.Game; using Poe2Trade.Screen; using Serilog; @@ -317,6 +319,38 @@ public partial class DebugViewModel : ObservableObject : $"Burst capture OFF — {_bot.FrameSaver.SavedCount} frames saved"; } + [RelayCommand] + private async Task DebugPortal() + { + try + { + DebugResult = "Portal debug: focusing game..."; + await _bot.Game.FocusGame(); + await Helpers.Sleep(Delays.PostFocus); + + // Open portal + await _bot.Game.PressPlus(); + await Helpers.Sleep(2000); + + // Run nameplate diff OCR with save + var savePath = Path.Combine("debug", "portal-ocr.png"); + Directory.CreateDirectory("debug"); + var portalRegion = new System.Drawing.Rectangle(1280 - 600, 100, 1200, 620); + var pos = await _bot.Inventory.FindAndClickNameplate( + "The Ardura Caravan", maxRetries: 3, retryDelayMs: 1500, + scanRegion: portalRegion, savePath: savePath); + + DebugResult = pos.HasValue + ? $"Found portal at ({pos.Value.X},{pos.Value.Y}), images saved to debug/" + : $"Portal not found, OCR images saved to debug/"; + } + catch (Exception ex) + { + DebugResult = $"Portal debug failed: {ex.Message}"; + Log.Error(ex, "Portal debug failed"); + } + } + [RelayCommand] private async Task ClickSalvage() { diff --git a/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs b/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs index 2aa3f55..680bf9b 100644 --- a/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/SettingsViewModel.cs @@ -173,7 +173,7 @@ public partial class SettingsViewModel : ObservableObject await Helpers.RandomDelay(800, 1200); // ANGE opens a dialog — click "Manage Shop" - var dialogRegion = new Region(1080, 600, 400, 300); + var dialogRegion = new Region(pos.Value.X, pos.Value.Y, 460, 600); var managePos = await _bot.Screen.FindTextInRegion(dialogRegion, "Manage"); if (managePos.HasValue) { diff --git a/src/Poe2Trade.Ui/Views/MainWindow.axaml b/src/Poe2Trade.Ui/Views/MainWindow.axaml index 3011849..fe976d1 100644 --- a/src/Poe2Trade.Ui/Views/MainWindow.axaml +++ b/src/Poe2Trade.Ui/Views/MainWindow.axaml @@ -327,6 +327,7 @@