diff --git a/assets/black-cathedral-door.png b/assets/black-cathedral-door.png new file mode 100644 index 0000000..f1fee0d Binary files /dev/null and b/assets/black-cathedral-door.png differ diff --git a/assets/return-the-ring.png b/assets/return-the-ring.png new file mode 100644 index 0000000..c76eae2 Binary files /dev/null and b/assets/return-the-ring.png differ diff --git a/src/Poe2Trade.Bot/BossRunExecutor.cs b/src/Poe2Trade.Bot/BossRunExecutor.cs index 8132ec9..2881713 100644 --- a/src/Poe2Trade.Bot/BossRunExecutor.cs +++ b/src/Poe2Trade.Bot/BossRunExecutor.cs @@ -3,38 +3,37 @@ using Poe2Trade.Core; using Poe2Trade.Game; using Poe2Trade.GameLog; using Poe2Trade.Inventory; +using Poe2Trade.Navigation; using Poe2Trade.Screen; using Serilog; namespace Poe2Trade.Bot; -public class BossRunExecutor +public class BossRunExecutor : GameExecutor { private static readonly string WellOfSoulsTemplate = Path.Combine("assets", "well-of-souls.png"); private static readonly string BlackCathedralTemplate = Path.Combine("assets", "black-cathedral.png"); private static readonly string InvitationTemplate = Path.Combine("assets", "invitation.png"); + private static readonly string CathedralDoorTemplate = Path.Combine("assets", "black-cathedral-door.png"); + private static readonly string ReturnTheRingTemplate = Path.Combine("assets", "return-the-ring.png"); private BossRunState _state = BossRunState.Idle; - private bool _stopped; - private readonly IGameController _game; - private readonly IScreenReader _screen; - private readonly IInventoryManager _inventory; private readonly IClientLogWatcher _logWatcher; - private readonly SavedSettings _config; private readonly BossDetector _bossDetector; + private readonly HudReader _hudReader; + private readonly NavigationExecutor _nav; public event Action? StateChanged; public BossRunExecutor(IGameController game, IScreenReader screen, IInventoryManager inventory, IClientLogWatcher logWatcher, SavedSettings config, - BossDetector bossDetector) + BossDetector bossDetector, HudReader hudReader, NavigationExecutor nav) + : base(game, screen, inventory, config) { - _game = game; - _screen = screen; - _inventory = inventory; _logWatcher = logWatcher; - _config = config; _bossDetector = bossDetector; + _hudReader = hudReader; + _nav = nav; } public BossRunState State => _state; @@ -45,9 +44,9 @@ public class BossRunExecutor StateChanged?.Invoke(s); } - public void Stop() + public override void Stop() { - _stopped = true; + base.Stop(); Log.Information("Boss run executor stop requested"); } @@ -100,6 +99,7 @@ public class BossRunExecutor await Fight(); if (_stopped) break; + SetState(BossRunState.Looting); await Loot(); if (_stopped) break; @@ -299,17 +299,251 @@ public class BossRunExecutor private async Task Fight() { SetState(BossRunState.Fighting); - Log.Information("[PLACEHOLDER] Fight phase - waiting for manual combat"); - // Placeholder: user handles combat manually for now - await Helpers.Sleep(1000); + Log.Information("Fight phase starting"); + + // Wait for arena to settle + await Helpers.Sleep(6000); + if (_stopped) return; + + // Find and click the cathedral door + Log.Information("Looking for cathedral door..."); + var door = await _screen.TemplateMatch(CathedralDoorTemplate); + if (door == null) + { + Log.Error("Could not find cathedral door template"); + return; + } + Log.Information("Found cathedral door at ({X},{Y}), clicking", door.X, door.Y); + await _game.LeftClickAt(door.X, door.Y); + + // Wait for cathedral interior to load + await Helpers.Sleep(12000); + if (_stopped) return; + + // Walk to fight area (world coords) + const double fightWorldX = -454; + const double fightWorldY = -332; + const double wellWorldX = -496; + const double wellWorldY = -378; + + await WalkToWorldPosition(fightWorldX, fightWorldY); + if (_stopped) return; + + // 3x fight-then-well loop + for (var phase = 1; phase <= 3; phase++) + { + if (_stopped) return; + Log.Information("=== Boss phase {Phase}/4 ===", phase); + + await AttackBossUntilGone(); + if (_stopped) return; + + // Walk to well and click it + Log.Information("Phase {Phase} done, walking to well", phase); + await WalkToWorldPosition(wellWorldX, wellWorldY); + // Click at screen center (well should be near character) + await _game.LeftClickAt(1280, 720); + await Helpers.Sleep(2000); + + // Walk back to fight position for next phase + await WalkToWorldPosition(fightWorldX, fightWorldY); + } + + // 4th fight - no well after + if (_stopped) return; + Log.Information("=== Boss phase 4/4 ==="); + await AttackBossUntilGone(); + if (_stopped) return; + + // Return the ring + Log.Information("Looking for Return the Ring..."); + var ring = await _screen.TemplateMatch(ReturnTheRingTemplate); + if (ring == null) + { + Log.Warning("Could not find Return the Ring template, retrying after 2s..."); + await Helpers.Sleep(2000); + ring = await _screen.TemplateMatch(ReturnTheRingTemplate); + } + if (ring != null) + { + Log.Information("Found Return the Ring at ({X},{Y}), clicking", ring.X, ring.Y); + await _game.LeftClickAt(ring.X, ring.Y); + await Helpers.Sleep(2000); + } + else + { + Log.Error("Could not find Return the Ring template"); + } + if (_stopped) return; + + // Walk up and press Q + Log.Information("Walking up and pressing Q"); + await _game.KeyDown(InputSender.VK.W); + await Helpers.Sleep(1500); + await _game.KeyUp(InputSender.VK.W); + await Helpers.Sleep(300); + await _game.PressKey(InputSender.VK.Q); + await Helpers.Sleep(500); + + // Spam L+R at position for 7s + Log.Information("Attacking at ring fight position (Q phase)"); + await AttackAtPosition(1280, 720, 7000); + if (_stopped) return; + + // Press E, spam L+R at same position for 7s + Log.Information("Pressing E and continuing attack"); + await _game.PressKey(InputSender.VK.E); + await Helpers.Sleep(500); + await AttackAtPosition(1280, 720, 7000); + + Log.Information("Fight complete"); } - private async Task Loot() + private async Task AttackBossUntilGone(int timeoutMs = 120_000) { - SetState(BossRunState.Looting); - Log.Information("[PLACEHOLDER] Loot phase - waiting for manual looting"); - // Placeholder: user handles looting manually for now - await Helpers.Sleep(1000); + // Move mouse to screen center initially + await _game.MoveMouseFast(1280, 720); + await Helpers.Sleep(200); + + var sw = Stopwatch.StartNew(); + var consecutiveMisses = 0; + + while (sw.ElapsedMilliseconds < timeoutMs) + { + if (_stopped) return; + + var snapshot = _bossDetector.Latest; + if (snapshot.Bosses.Count > 0) + { + consecutiveMisses = 0; + var boss = snapshot.Bosses[0]; + + // Check mana before attacking + var hud = _hudReader.Current; + if (hud.ManaPct < 0.80f) + { + await Helpers.Sleep(200); + continue; + } + + // Move to boss and attack + var targetX = boss.Cx + Rng.Next(-10, 11); + var targetY = boss.Cy + Rng.Next(-10, 11); + await _game.MoveMouseFast(targetX, targetY); + + _game.LeftMouseDown(); + await Helpers.Sleep(Rng.Next(30, 50)); + _game.LeftMouseUp(); + await Helpers.Sleep(Rng.Next(20, 40)); + _game.RightMouseDown(); + await Helpers.Sleep(Rng.Next(30, 50)); + _game.RightMouseUp(); + + await Helpers.Sleep(Rng.Next(100, 150)); + } + else + { + consecutiveMisses++; + if (consecutiveMisses >= 15) + { + Log.Information("Boss gone after {Ms}ms ({Misses} consecutive misses)", + sw.ElapsedMilliseconds, consecutiveMisses); + return; + } + await Helpers.Sleep(200); + } + } + + Log.Warning("AttackBossUntilGone timed out after {Ms}ms", timeoutMs); + } + + private async Task AttackAtPosition(int x, int y, int durationMs) + { + var sw = Stopwatch.StartNew(); + while (sw.ElapsedMilliseconds < durationMs) + { + if (_stopped) return; + + var targetX = x + Rng.Next(-20, 21); + var targetY = y + Rng.Next(-20, 21); + await _game.MoveMouseFast(targetX, targetY); + + _game.LeftMouseDown(); + await Helpers.Sleep(Rng.Next(30, 50)); + _game.LeftMouseUp(); + await Helpers.Sleep(Rng.Next(20, 40)); + _game.RightMouseDown(); + await Helpers.Sleep(Rng.Next(30, 50)); + _game.RightMouseUp(); + + await Helpers.Sleep(Rng.Next(100, 150)); + } + } + + /// + /// Walk to a world position using WASD keys, checking minimap position each iteration. + /// + private async Task WalkToWorldPosition(double worldX, double worldY, int timeoutMs = 10000, double arrivalDist = 15) + { + Log.Information("Walking to world ({X:F0},{Y:F0})", worldX, worldY); + + var sw = Stopwatch.StartNew(); + var heldKeys = new HashSet(); + + try + { + while (sw.ElapsedMilliseconds < timeoutMs) + { + if (_stopped) break; + + var pos = _nav.WorldPosition; + var dx = worldX - pos.X; + var dy = worldY - pos.Y; + var dist = Math.Sqrt(dx * dx + dy * dy); + + if (dist <= arrivalDist) + { + Log.Information("Arrived at ({X:F0},{Y:F0}), dist={Dist:F0}", pos.X, pos.Y, dist); + break; + } + + // Normalize direction + var len = Math.Sqrt(dx * dx + dy * dy); + var dirX = dx / len; + var dirY = dy / len; + + // Map direction to WASD keys + var wanted = new HashSet(); + if (dirY < -0.3) wanted.Add(InputSender.VK.W); // up + if (dirY > 0.3) wanted.Add(InputSender.VK.S); // down + if (dirX < -0.3) wanted.Add(InputSender.VK.A); // left + if (dirX > 0.3) wanted.Add(InputSender.VK.D); // right + + // Release keys no longer needed + foreach (var key in heldKeys.Except(wanted).ToList()) + { + await _game.KeyUp(key); + heldKeys.Remove(key); + } + // Press new keys + foreach (var key in wanted.Except(heldKeys).ToList()) + { + await _game.KeyDown(key); + heldKeys.Add(key); + } + + await Helpers.Sleep(100); + } + + if (sw.ElapsedMilliseconds >= timeoutMs) + Log.Warning("WalkToWorldPosition timed out after {Ms}ms", timeoutMs); + } + finally + { + // Release all held keys + foreach (var key in heldKeys) + await _game.KeyUp(key); + } } private async Task ReturnHome() @@ -408,122 +642,4 @@ public class BossRunExecutor Log.Information("Loot stored"); } - - private async Task WalkAndMatch(string templatePath, int vk1, int vk2, - int timeoutMs = 15000, int closeRadius = 350) - { - const int screenCx = 2560 / 2; - const int screenCy = 1440 / 2; - - await _game.KeyDown(vk1); - await _game.KeyDown(vk2); - try - { - var sw = Stopwatch.StartNew(); - bool spotted = false; - while (sw.ElapsedMilliseconds < timeoutMs) - { - if (_stopped) return null; - var match = await _screen.TemplateMatch(templatePath); - if (match == null) - { - await Helpers.Sleep(500); - continue; - } - - var dx = match.X - screenCx; - var dy = match.Y - screenCy; - var dist = Math.Sqrt(dx * dx + dy * dy); - - if (!spotted) - { - Log.Information("Template spotted at ({X},{Y}), dist={Dist:F0}px from center, approaching...", - match.X, match.Y, dist); - spotted = true; - } - - if (dist <= closeRadius) - { - Log.Information("Close enough at ({X},{Y}), dist={Dist:F0}px, stopping", match.X, match.Y, dist); - - // Stop, settle, re-match for accurate position - await _game.KeyUp(vk2); - await _game.KeyUp(vk1); - await Helpers.Sleep(300); - - var fresh = await _screen.TemplateMatch(templatePath); - if (fresh != null) - { - Log.Information("Final position at ({X},{Y})", fresh.X, fresh.Y); - return fresh; - } - Log.Warning("Re-match failed, using last known position"); - return match; - } - - await Helpers.Sleep(200); - } - Log.Error("WalkAndMatch timed out after {Ms}ms (spotted={Spotted})", timeoutMs, spotted); - return null; - } - finally - { - await _game.KeyUp(vk2); - await _game.KeyUp(vk1); - } - } - - private (StashTabInfo? Tab, StashTabInfo? Folder) ResolveTabPath(string tabPath) - { - if (string.IsNullOrEmpty(tabPath) || _config.StashCalibration == null) - return (null, null); - - var parts = tabPath.Split('/'); - if (parts.Length == 1) - { - // Simple tab name - var tab = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0]); - return (tab, null); - } - - if (parts.Length == 2) - { - // Folder/SubTab - var folder = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0] && t.IsFolder); - if (folder == null) return (null, null); - var subTab = folder.SubTabs.FirstOrDefault(t => t.Name == parts[1]); - return (subTab, folder); - } - - return (null, null); - } - - private async Task RecoverToHideout() - { - try - { - Log.Information("Recovering: escaping and going to hideout"); - await _game.FocusGame(); - await _game.PressEscape(); - await Helpers.Sleep(Delays.PostEscape); - await _game.PressEscape(); - await Helpers.Sleep(Delays.PostEscape); - - var arrived = await _inventory.WaitForAreaTransition( - _config.TravelTimeoutMs, () => _game.GoToHideout()); - if (arrived) - { - _inventory.SetLocation(true); - Log.Information("Recovery: arrived at hideout"); - } - else - { - Log.Warning("Recovery: timed out going to hideout"); - } - } - catch (Exception ex) - { - Log.Error(ex, "Recovery failed"); - } - } } diff --git a/src/Poe2Trade.Bot/BotOrchestrator.cs b/src/Poe2Trade.Bot/BotOrchestrator.cs index 8def5e6..7797666 100644 --- a/src/Poe2Trade.Bot/BotOrchestrator.cs +++ b/src/Poe2Trade.Bot/BotOrchestrator.cs @@ -74,7 +74,9 @@ public class BotOrchestrator : IAsyncDisposable GameState = new GameStateDetector(); HudReader = new HudReader(); EnemyDetector = new EnemyDetector(); + EnemyDetector.Enabled = true; BossDetector = new BossDetector(); + BossDetector.Enabled = true; FrameSaver = new FrameSaver(); // Register on shared pipeline @@ -89,7 +91,7 @@ public class BotOrchestrator : IAsyncDisposable Navigation = new NavigationExecutor(game, pipelineService.Pipeline, minimapCapture, enemyDetector: EnemyDetector); - BossRunExecutor = new BossRunExecutor(game, screen, inventory, logWatcher, store.Settings, BossDetector); + BossRunExecutor = new BossRunExecutor(game, screen, inventory, logWatcher, store.Settings, BossDetector, HudReader, Navigation); logWatcher.AreaEntered += area => { @@ -111,13 +113,7 @@ public class BotOrchestrator : IAsyncDisposable if (BossZones.TryGetValue(area, out var boss)) { BossDetector.SetBoss(boss); - BossDetector.Enabled = true; - Log.Information("Boss zone detected: {Area} → enabling {Boss} detector", area, boss); - } - else if (BossDetector.Enabled) - { - BossDetector.Enabled = false; - Log.Information("Left boss zone → disabling boss detector"); + Log.Information("Boss zone detected: {Area} → switching to {Boss} model", area, boss); } } diff --git a/src/Poe2Trade.Bot/GameExecutor.cs b/src/Poe2Trade.Bot/GameExecutor.cs new file mode 100644 index 0000000..5bd5d1e --- /dev/null +++ b/src/Poe2Trade.Bot/GameExecutor.cs @@ -0,0 +1,222 @@ +using System.Diagnostics; +using Poe2Trade.Core; +using Poe2Trade.Game; +using Poe2Trade.Inventory; +using Poe2Trade.Screen; +using Serilog; + +namespace Poe2Trade.Bot; + +/// +/// Base class for game executors that interact with the game world. +/// Provides shared utilities: loot pickup, recovery, walking, stash tab resolution. +/// +public abstract class GameExecutor +{ + protected static readonly Random Rng = new(); + + protected readonly IGameController _game; + protected readonly IScreenReader _screen; + protected readonly IInventoryManager _inventory; + protected readonly SavedSettings _config; + protected volatile bool _stopped; + + protected GameExecutor(IGameController game, IScreenReader screen, + IInventoryManager inventory, SavedSettings config) + { + _game = game; + _screen = screen; + _inventory = inventory; + _config = config; + } + + public virtual void Stop() + { + _stopped = true; + } + + // ------ Loot pickup ------ + + // Tiers to skip (noise, low-value, or hidden by filter) + private static readonly HashSet SkipTiers = ["unknown", "gold"]; + + public async Task Loot() + { + Log.Information("Starting loot pickup"); + + const int maxRounds = 5; + for (var round = 0; round < maxRounds; round++) + { + if (_stopped) return; + + // Move mouse out of the way so it doesn't cover labels + _game.MoveMouseInstant(0, 1440); + await Helpers.Sleep(100); + + // Hold Alt to ensure all labels are visible, then capture + await _game.KeyDown(InputSender.VK.MENU); + await Helpers.Sleep(250); + var capture = _screen.CaptureRawBitmap(); + + // Detect magenta-bordered labels directly (no diff needed) + var labels = _screen.DetectLootLabels(capture, capture); + capture.Dispose(); + + // Filter out noise and unwanted tiers + var pickups = labels.Where(l => !SkipTiers.Contains(l.Tier)).ToList(); + + if (pickups.Count == 0) + { + await _game.KeyUp(InputSender.VK.MENU); + Log.Information("No loot labels in round {Round} (total detected: {Total}, filtered: {Filtered})", + round + 1, labels.Count, labels.Count - pickups.Count); + break; + } + + Log.Information("Round {Round}: {Count} loot labels ({Skipped} skipped)", + round + 1, pickups.Count, labels.Count - pickups.Count); + + foreach (var skip in labels.Where(l => SkipTiers.Contains(l.Tier))) + Log.Debug("Skipped: tier={Tier} color=({R},{G},{B}) at ({X},{Y})", + skip.Tier, skip.AvgR, skip.AvgG, skip.AvgB, skip.CenterX, skip.CenterY); + + // Click each label center (Alt still held so labels visible) + foreach (var label in pickups) + { + if (_stopped) break; + + Log.Information("Picking up: tier={Tier} color=({R},{G},{B}) at ({X},{Y})", + label.Tier, label.AvgR, label.AvgG, label.AvgB, label.CenterX, label.CenterY); + await _game.LeftClickAt(label.CenterX, label.CenterY); + await Helpers.Sleep(300); + } + + await _game.KeyUp(InputSender.VK.MENU); + await Helpers.Sleep(500); + } + + Log.Information("Loot pickup complete"); + } + + // ------ Recovery ------ + + protected async Task RecoverToHideout() + { + try + { + Log.Information("Recovering: escaping and going to hideout"); + await _game.FocusGame(); + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + await _game.PressEscape(); + await Helpers.Sleep(Delays.PostEscape); + + var arrived = await _inventory.WaitForAreaTransition( + _config.TravelTimeoutMs, () => _game.GoToHideout()); + if (arrived) + { + _inventory.SetLocation(true); + Log.Information("Recovery: arrived at hideout"); + } + else + { + Log.Warning("Recovery: timed out going to hideout"); + } + } + catch (Exception ex) + { + Log.Error(ex, "Recovery failed"); + } + } + + // ------ Walk + template match ------ + + protected async Task WalkAndMatch(string templatePath, int vk1, int vk2, + int timeoutMs = 15000, int closeRadius = 350) + { + const int screenCx = 2560 / 2; + const int screenCy = 1440 / 2; + + await _game.KeyDown(vk1); + await _game.KeyDown(vk2); + try + { + var sw = Stopwatch.StartNew(); + bool spotted = false; + while (sw.ElapsedMilliseconds < timeoutMs) + { + if (_stopped) return null; + var match = await _screen.TemplateMatch(templatePath); + if (match == null) + { + await Helpers.Sleep(500); + continue; + } + + var dx = match.X - screenCx; + var dy = match.Y - screenCy; + var dist = Math.Sqrt(dx * dx + dy * dy); + + if (!spotted) + { + Log.Information("Template spotted at ({X},{Y}), dist={Dist:F0}px from center, approaching...", + match.X, match.Y, dist); + spotted = true; + } + + if (dist <= closeRadius) + { + Log.Information("Close enough at ({X},{Y}), dist={Dist:F0}px, stopping", match.X, match.Y, dist); + + // Stop, settle, re-match for accurate position + await _game.KeyUp(vk2); + await _game.KeyUp(vk1); + await Helpers.Sleep(300); + + var fresh = await _screen.TemplateMatch(templatePath); + if (fresh != null) + { + Log.Information("Final position at ({X},{Y})", fresh.X, fresh.Y); + return fresh; + } + Log.Warning("Re-match failed, using last known position"); + return match; + } + + await Helpers.Sleep(200); + } + Log.Error("WalkAndMatch timed out after {Ms}ms (spotted={Spotted})", timeoutMs, spotted); + return null; + } + finally + { + await _game.KeyUp(vk2); + await _game.KeyUp(vk1); + } + } + + // ------ Stash tab resolution ------ + + protected (StashTabInfo? Tab, StashTabInfo? Folder) ResolveTabPath(string tabPath) + { + if (string.IsNullOrEmpty(tabPath) || _config.StashCalibration == null) + return (null, null); + + var parts = tabPath.Split('/'); + if (parts.Length == 1) + { + var tab = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0]); + return (tab, null); + } + + if (parts.Length == 2) + { + var folder = _config.StashCalibration.Tabs.FirstOrDefault(t => t.Name == parts[0] && t.IsFolder); + if (folder == null) return (null, null); + var subTab = folder.SubTabs.FirstOrDefault(t => t.Name == parts[1]); + return (subTab, folder); + } + + return (null, null); + } +} diff --git a/src/Poe2Trade.Game/InputSender.cs b/src/Poe2Trade.Game/InputSender.cs index c469c97..7c93bc2 100644 --- a/src/Poe2Trade.Game/InputSender.cs +++ b/src/Poe2Trade.Game/InputSender.cs @@ -34,6 +34,8 @@ public class InputSender public const int W = 0x57; public const int S = 0x53; public const int D = 0x44; + public const int E = 0x45; + public const int Q = 0x51; public const int Z = 0x5A; } diff --git a/src/Poe2Trade.Navigation/NavigationExecutor.cs b/src/Poe2Trade.Navigation/NavigationExecutor.cs index 5470900..9bfbe2e 100644 --- a/src/Poe2Trade.Navigation/NavigationExecutor.cs +++ b/src/Poe2Trade.Navigation/NavigationExecutor.cs @@ -491,6 +491,7 @@ public class NavigationExecutor : IDisposable public bool IsExploring => _state != NavigationState.Idle && _state != NavigationState.Completed && _state != NavigationState.Failed; public MapPosition Position => _worldMap.Position; + public MapPosition WorldPosition => _worldMap.WorldPosition; public byte[] GetMapSnapshot() => _worldMap.GetMapSnapshot(); public byte[] GetViewportSnapshot(int viewSize = 400) { diff --git a/src/Poe2Trade.Navigation/WorldMap.cs b/src/Poe2Trade.Navigation/WorldMap.cs index f697d57..79ed437 100644 --- a/src/Poe2Trade.Navigation/WorldMap.cs +++ b/src/Poe2Trade.Navigation/WorldMap.cs @@ -21,7 +21,19 @@ public class WorldMap : IDisposable private readonly List<(Point Pos, long LastSeenMs)> _checkpointsOn = []; private const int CheckpointDedupRadius = 20; + // World origin: cumulative offset from canvas (0,0) to world (0,0). + // World coords = canvas coords - _worldOrigin. Stable across canvas growth. + private double _worldOriginX; + private double _worldOriginY; + public MapPosition Position => _position; + + /// + /// Player position in stable world coordinates (invariant to canvas growth). + /// World (0,0) = where the player spawned. + /// + public MapPosition WorldPosition => new(_position.X - _worldOriginX, _position.Y - _worldOriginY); + public bool LastMatchSucceeded { get; private set; } public int CanvasSize => _canvasSize; internal List? LastBfsPath => _pathFinder.LastResult?.Path; @@ -36,6 +48,9 @@ public class WorldMap : IDisposable _canvas = new Mat(_canvasSize, _canvasSize, MatType.CV_8UC1, Scalar.Black); _confidence = new Mat(_canvasSize, _canvasSize, MatType.CV_16SC1, Scalar.Black); _position = new MapPosition(_canvasSize / 2.0, _canvasSize / 2.0); + // World origin = initial player position, so WorldPosition starts at (0,0) + _worldOriginX = _position.X; + _worldOriginY = _position.Y; } private void EnsureCapacity() @@ -63,7 +78,17 @@ public class WorldMap : IDisposable _confidence = newConf; _canvasSize = newSize; _position = new MapPosition(_position.X + offset, _position.Y + offset); - Log.Information("Canvas grown: {Old}→{New}, offset={Offset}", oldSize, newSize, offset); + _worldOriginX += offset; + _worldOriginY += offset; + + // Shift checkpoint canvas coordinates + for (var i = 0; i < _checkpointsOff.Count; i++) + _checkpointsOff[i] = (new Point(_checkpointsOff[i].Pos.X + offset, _checkpointsOff[i].Pos.Y + offset), _checkpointsOff[i].LastSeenMs); + for (var i = 0; i < _checkpointsOn.Count; i++) + _checkpointsOn[i] = (new Point(_checkpointsOn[i].Pos.X + offset, _checkpointsOn[i].Pos.Y + offset), _checkpointsOn[i].LastSeenMs); + + Log.Information("Canvas grown: {Old}→{New}, offset={Offset}, worldOrigin=({Ox:F0},{Oy:F0})", + oldSize, newSize, offset, _worldOriginX, _worldOriginY); } /// @@ -619,6 +644,14 @@ public class WorldMap : IDisposable return best; } + /// Convert world coordinates to canvas coordinates. + public MapPosition WorldToCanvas(double worldX, double worldY) => + new(worldX + _worldOriginX, worldY + _worldOriginY); + + /// Convert canvas coordinates to world coordinates. + public MapPosition CanvasToWorld(double canvasX, double canvasY) => + new(canvasX - _worldOriginX, canvasY - _worldOriginY); + public void Reset() { _canvas.Dispose(); @@ -629,6 +662,8 @@ public class WorldMap : IDisposable _prevWallMask?.Dispose(); _prevWallMask = null; _position = new MapPosition(_canvasSize / 2.0, _canvasSize / 2.0); + _worldOriginX = _position.X; + _worldOriginY = _position.Y; _frameCount = 0; _consecutiveMatchFails = 0; LastMatchSucceeded = false; diff --git a/src/Poe2Trade.Screen/IScreenReader.cs b/src/Poe2Trade.Screen/IScreenReader.cs index b9b9b95..fe35ebc 100644 --- a/src/Poe2Trade.Screen/IScreenReader.cs +++ b/src/Poe2Trade.Screen/IScreenReader.cs @@ -18,6 +18,8 @@ public interface IScreenReader : IDisposable Task DiffOcr(string? savePath = null, Region? region = null); Task TemplateMatch(string templatePath, Region? region = null); Task NameplateDiffOcr(System.Drawing.Bitmap reference, System.Drawing.Bitmap current); + void SetLootBaseline(System.Drawing.Bitmap frame); + List DetectLootLabels(System.Drawing.Bitmap reference, System.Drawing.Bitmap current); System.Drawing.Bitmap CaptureRawBitmap(); Task SaveScreenshot(string path); Task SaveRegion(Region region, string path); diff --git a/src/Poe2Trade.Screen/LootLabel.cs b/src/Poe2Trade.Screen/LootLabel.cs new file mode 100644 index 0000000..0ae2d0b --- /dev/null +++ b/src/Poe2Trade.Screen/LootLabel.cs @@ -0,0 +1,110 @@ +namespace Poe2Trade.Screen; + +/// +/// A detected loot label on screen with its position and classified tier. +/// +public record LootLabel(int CenterX, int CenterY, int Width, int Height, string Tier, byte AvgR, byte AvgG, byte AvgB); + +/// +/// Classifies loot label background colors to NeverSink filter tiers +/// by matching against known filter color palette. +/// +public static class LootColorClassifier +{ + private record ColorEntry(byte R, byte G, byte B, string Tier); + + // Background colors from NeverSink's Uber Strict filter + private static readonly ColorEntry[] KnownBgColors = + [ + // S-tier (apex): white bg + new(255, 255, 255, "S"), + // A-tier: red bg, white text + new(245, 105, 90, "A"), + // C-tier: orange bg + new(245, 139, 87, "C"), + // D-tier: yellow bg + new(240, 180, 100, "D"), + // E-tier: text-only, yellowish + new(240, 207, 132, "E"), + + // Unique high: brown bg + new(188, 96, 37, "unique"), + // Unique T3: dark red bg + new(53, 13, 13, "unique-low"), + + // Exotic bases: dark green bg + new(0, 75, 30, "exotic"), + // Identified mods: dark purple bg + new(47, 0, 74, "exotic-mod"), + + // Rare jewellery: olive bg + new(75, 75, 0, "rare-jewellery"), + + // Fragments: bright purple bg + new(220, 0, 255, "fragment"), + // Fragments lower: light purple bg + new(180, 75, 225, "fragment"), + // Fragment splinter: dark purple bg + new(50, 0, 75, "fragment"), + + // Maps special: lavender bg + new(235, 220, 245, "map"), + // Maps regular high: light grey bg + new(235, 235, 235, "map"), + // Maps regular: grey bg + new(200, 200, 200, "map"), + + // Crafting magic: dark blue-purple bg + new(30, 0, 70, "crafting"), + + // Gems: cyan text (20,240,240) - no bg + new(20, 240, 240, "gem"), + // Gems: dark blue bg + new(6, 0, 60, "gem"), + + // Flasks/charms: dark green bg + new(10, 60, 40, "flask"), + + // Currency artifact: dark brown bg + new(76, 51, 12, "artifact"), + + // Socketables (runes): orange-tan bg + new(220, 175, 132, "socketable"), + + // Gold drops: gold/yellow text + new(180, 160, 80, "gold"), + new(200, 180, 100, "gold"), + + // Pink/magenta catch-all (e.g. boss-specific drops like invitations) + new(255, 0, 255, "special"), + new(220, 50, 220, "special"), + ]; + + private const double MaxDistance = 50.0; + + /// + /// Classify an average RGB color to the closest NeverSink filter tier. + /// Returns "unknown" if no known color is within MaxDistance. + /// + public static string Classify(byte avgR, byte avgG, byte avgB) + { + double bestDist = double.MaxValue; + string bestTier = "unknown"; + + foreach (var entry in KnownBgColors) + { + double dr = avgR - entry.R; + double dg = avgG - entry.G; + double db = avgB - entry.B; + double dist = Math.Sqrt(dr * dr + dg * dg + db * db); + + if (dist < bestDist) + { + bestDist = dist; + bestTier = entry.Tier; + } + } + + return bestDist <= MaxDistance ? bestTier : "unknown"; + } +} diff --git a/src/Poe2Trade.Screen/ScreenReader.cs b/src/Poe2Trade.Screen/ScreenReader.cs index 77f6d82..ae378a9 100644 --- a/src/Poe2Trade.Screen/ScreenReader.cs +++ b/src/Poe2Trade.Screen/ScreenReader.cs @@ -1,10 +1,12 @@ using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; -using Poe2Trade.Core; +using OpenCvSharp; using OpenCvSharp.Extensions; +using Poe2Trade.Core; using Serilog; using Region = Poe2Trade.Core.Region; +using Size = OpenCvSharp.Size; namespace Poe2Trade.Screen; @@ -320,6 +322,78 @@ public class ScreenReader : IScreenReader return boxes; } + // -- Loot label detection (magenta background) -- + // + // All loot labels: white border, magenta (255,0,255) background, black text. + // Magenta never appears in the game world → detect directly, no diff needed. + + public void SetLootBaseline(Bitmap frame) { } + + public List DetectLootLabels(Bitmap reference, Bitmap current) + { + using var mat = BitmapConverter.ToMat(current); + if (mat.Channels() == 4) + Cv2.CvtColor(mat, mat, ColorConversionCodes.BGRA2BGR); + + // Mask magenta background pixels (BGR: B≈255, G≈0, R≈255) + using var mask = new Mat(); + Cv2.InRange(mat, new Scalar(200, 0, 200), new Scalar(255, 60, 255), mask); + + // Morph close fills text gaps within a label + // Height=2 bridges line gaps within multi-line labels but not between separate labels + using var kernel = Cv2.GetStructuringElement(MorphShapes.Rect, new Size(12, 2)); + using var closed = new Mat(); + Cv2.MorphologyEx(mask, closed, MorphTypes.Close, kernel); + + // Save debug images + try + { + Cv2.ImWrite("debug_loot_mask.png", mask); + Cv2.ImWrite("debug_loot_closed.png", closed); + current.Save("debug_loot_capture.png", System.Drawing.Imaging.ImageFormat.Png); + Log.Information("Saved debug images: debug_loot_mask.png, debug_loot_closed.png, debug_loot_capture.png"); + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to save debug images"); + } + + Cv2.FindContours(closed, out var contours, out _, + RetrievalModes.External, ContourApproximationModes.ApproxSimple); + + Log.Information("DetectLootLabels: {N} magenta contours", contours.Length); + + const int minW = 40, maxW = 600; + const int minH = 8, maxH = 100; + const double minAspect = 1.5; + int yMax = mat.Height - 210; + + var labels = new List(); + foreach (var contour in contours) + { + var box = Cv2.BoundingRect(contour); + double aspect = box.Height > 0 ? (double)box.Width / box.Height : 0; + + if (box.Width < minW || box.Width > maxW || + box.Height < minH || box.Height > maxH || + aspect < minAspect || + box.Y < 65 || box.Y + box.Height > yMax) + { + Log.Information("Rejected contour: ({X},{Y}) {W}x{H} aspect={Aspect:F1} yMax={YMax}", + box.X, box.Y, box.Width, box.Height, aspect, yMax); + continue; + } + + int cx = box.X + box.Width / 2; + int cy = box.Y + box.Height / 2; + + Log.Information("Label at ({X},{Y}) {W}x{H}", box.X, box.Y, box.Width, box.Height); + labels.Add(new LootLabel(cx, cy, box.Width, box.Height, "loot", 255, 0, 255)); + } + + return labels; + } + public void Dispose() => _pythonBridge.Dispose(); // -- OCR text matching -- diff --git a/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs b/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs index ee7bab2..d03d1a2 100644 --- a/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs +++ b/src/Poe2Trade.Ui/Overlay/D2dOverlay.cs @@ -189,7 +189,7 @@ public sealed class D2dOverlay InferenceMs: detection.InferenceMs, Hud: _bot.HudReader.Current, NavState: _bot.Navigation.State, - NavPosition: _bot.Navigation.Position, + NavPosition: _bot.Navigation.WorldPosition, IsExploring: _bot.Navigation.IsExploring, ShowHudDebug: _bot.Store.Settings.ShowHudDebug, Fps: fps, diff --git a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs index b403995..6e350ec 100644 --- a/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/DebugViewModel.cs @@ -323,4 +323,22 @@ public partial class DebugViewModel : ObservableObject catch (Exception ex) { DebugResult = $"Failed: {ex.Message}"; } } + [RelayCommand] + private async Task LootTest() + { + try + { + DebugResult = "Loot test: focusing game..."; + await _bot.Game.FocusGame(); + await Task.Delay(300); + await _bot.BossRunExecutor.Loot(); + DebugResult = "Loot test: complete"; + } + catch (Exception ex) + { + DebugResult = $"Loot test failed: {ex.Message}"; + Log.Error(ex, "Loot test failed"); + } + } + } diff --git a/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs b/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs index 2b83730..1798a1e 100644 --- a/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs +++ b/src/Poe2Trade.Ui/ViewModels/MappingViewModel.cs @@ -16,7 +16,6 @@ public partial class MappingViewModel : ObservableObject, IDisposable [ObservableProperty] private MapType _selectedMapType; [ObservableProperty] private bool _isFrameSaverEnabled; [ObservableProperty] private int _framesSaved; - [ObservableProperty] private bool _isDetectionEnabled; [ObservableProperty] private int _enemiesDetected; [ObservableProperty] private float _inferenceMs; [ObservableProperty] private bool _hasModel; @@ -106,12 +105,6 @@ public partial class MappingViewModel : ObservableObject, IDisposable _bot.FrameSaver.Enabled = value; } - partial void OnIsDetectionEnabledChanged(bool value) - { - _bot.EnemyDetector.Enabled = value; - _bot.BossDetector.Enabled = value; - } - private void OnDetectionUpdated(DetectionSnapshot snapshot) { Dispatcher.UIThread.Post(() => diff --git a/src/Poe2Trade.Ui/Views/MainWindow.axaml b/src/Poe2Trade.Ui/Views/MainWindow.axaml index 3ea8a1e..b420fcd 100644 --- a/src/Poe2Trade.Ui/Views/MainWindow.axaml +++ b/src/Poe2Trade.Ui/Views/MainWindow.axaml @@ -287,9 +287,6 @@ -