switched to new way :)
This commit is contained in:
parent
b03a2a25f1
commit
f22d182c8f
30 changed files with 0 additions and 0 deletions
139
src-old/game/GameController.ts
Normal file
139
src-old/game/GameController.ts
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
import { WindowManager } from './WindowManager.js';
|
||||
import { InputSender, VK } from './InputSender.js';
|
||||
import { sleep, randomDelay } from '../util/sleep.js';
|
||||
import { writeClipboard } from '../util/clipboard.js';
|
||||
import { logger } from '../util/logger.js';
|
||||
import type { Config } from '../types.js';
|
||||
|
||||
export class GameController {
|
||||
private windowManager: WindowManager;
|
||||
private inputSender: InputSender;
|
||||
|
||||
constructor(config: Config) {
|
||||
this.windowManager = new WindowManager(config.poe2WindowTitle);
|
||||
this.inputSender = new InputSender();
|
||||
}
|
||||
|
||||
async focusGame(): Promise<boolean> {
|
||||
const result = this.windowManager.focusWindow();
|
||||
if (result) {
|
||||
await sleep(300); // Wait for window to actually focus
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
isGameFocused(): boolean {
|
||||
return this.windowManager.isGameFocused();
|
||||
}
|
||||
|
||||
getWindowRect() {
|
||||
return this.windowManager.getWindowRect();
|
||||
}
|
||||
|
||||
async sendChat(message: string): Promise<void> {
|
||||
logger.info({ message }, 'Sending chat message');
|
||||
|
||||
// Open chat
|
||||
await this.inputSender.pressKey(VK.RETURN);
|
||||
await randomDelay(100, 200);
|
||||
|
||||
// Clear any existing text
|
||||
await this.inputSender.selectAll();
|
||||
await sleep(50);
|
||||
await this.inputSender.pressKey(VK.BACK);
|
||||
await sleep(50);
|
||||
|
||||
// Type the message
|
||||
await this.inputSender.typeText(message);
|
||||
await randomDelay(50, 100);
|
||||
|
||||
// Send
|
||||
await this.inputSender.pressKey(VK.RETURN);
|
||||
await sleep(100);
|
||||
}
|
||||
|
||||
async sendChatViaPaste(message: string): Promise<void> {
|
||||
logger.info({ message }, 'Sending chat message via paste');
|
||||
|
||||
// Copy message to clipboard
|
||||
writeClipboard(message);
|
||||
await sleep(50);
|
||||
|
||||
// Open chat
|
||||
await this.inputSender.pressKey(VK.RETURN);
|
||||
await randomDelay(100, 200);
|
||||
|
||||
// Clear any existing text
|
||||
await this.inputSender.selectAll();
|
||||
await sleep(50);
|
||||
await this.inputSender.pressKey(VK.BACK);
|
||||
await sleep(50);
|
||||
|
||||
// Paste
|
||||
await this.inputSender.paste();
|
||||
await randomDelay(100, 200);
|
||||
|
||||
// Send
|
||||
await this.inputSender.pressKey(VK.RETURN);
|
||||
await sleep(100);
|
||||
}
|
||||
|
||||
async goToHideout(): Promise<void> {
|
||||
logger.info('Sending /hideout command');
|
||||
await this.sendChatViaPaste('/hideout');
|
||||
}
|
||||
|
||||
async ctrlRightClickAt(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.ctrlRightClick(x, y);
|
||||
}
|
||||
|
||||
async moveMouseTo(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.moveMouse(x, y);
|
||||
}
|
||||
|
||||
moveMouseInstant(x: number, y: number): void {
|
||||
this.inputSender.moveMouseInstant(x, y);
|
||||
}
|
||||
|
||||
async moveMouseFast(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.moveMouseFast(x, y);
|
||||
}
|
||||
|
||||
async leftClickAt(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.leftClick(x, y);
|
||||
}
|
||||
|
||||
async rightClickAt(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.rightClick(x, y);
|
||||
}
|
||||
|
||||
async holdAlt(): Promise<void> {
|
||||
await this.inputSender.keyDown(VK.MENU);
|
||||
}
|
||||
|
||||
async releaseAlt(): Promise<void> {
|
||||
await this.inputSender.keyUp(VK.MENU);
|
||||
}
|
||||
|
||||
async pressEscape(): Promise<void> {
|
||||
await this.inputSender.pressKey(VK.ESCAPE);
|
||||
}
|
||||
|
||||
async openInventory(): Promise<void> {
|
||||
logger.info('Opening inventory');
|
||||
await this.inputSender.pressKey(VK.I);
|
||||
await sleep(300);
|
||||
}
|
||||
|
||||
async ctrlLeftClickAt(x: number, y: number): Promise<void> {
|
||||
await this.inputSender.ctrlLeftClick(x, y);
|
||||
}
|
||||
|
||||
async holdCtrl(): Promise<void> {
|
||||
await this.inputSender.keyDown(VK.CONTROL);
|
||||
}
|
||||
|
||||
async releaseCtrl(): Promise<void> {
|
||||
await this.inputSender.keyUp(VK.CONTROL);
|
||||
}
|
||||
}
|
||||
161
src-old/game/GridReader.ts
Normal file
161
src-old/game/GridReader.ts
Normal file
|
|
@ -0,0 +1,161 @@
|
|||
import { logger } from '../util/logger.js';
|
||||
import type { OcrDaemon, GridItem, GridMatch } from './OcrDaemon.js';
|
||||
import type { Region } from '../types.js';
|
||||
|
||||
// ── Grid type definitions ───────────────────────────────────────────────────
|
||||
|
||||
export interface GridLayout {
|
||||
region: Region;
|
||||
cols: number;
|
||||
rows: number;
|
||||
}
|
||||
|
||||
export interface CellCoord {
|
||||
row: number;
|
||||
col: number;
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface ScanResult {
|
||||
layout: GridLayout;
|
||||
occupied: CellCoord[];
|
||||
items: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
}
|
||||
|
||||
// ── Calibrated grid layouts (2560×1440) ─────────────────────────────────────
|
||||
|
||||
export const GRID_LAYOUTS: Record<string, GridLayout> = {
|
||||
/** Player inventory — always 12×5, right side (below equipment slots) */
|
||||
inventory: {
|
||||
region: { x: 1696, y: 788, width: 840, height: 350 },
|
||||
cols: 12,
|
||||
rows: 5,
|
||||
},
|
||||
/** Personal stash 12×12 — left side, tab not in folder */
|
||||
stash12: {
|
||||
region: { x: 23, y: 169, width: 840, height: 840 },
|
||||
cols: 12,
|
||||
rows: 12,
|
||||
},
|
||||
/** Personal stash 12×12 — left side, tab in folder */
|
||||
stash12_folder: {
|
||||
region: { x: 23, y: 216, width: 840, height: 840 },
|
||||
cols: 12,
|
||||
rows: 12,
|
||||
},
|
||||
/** Personal stash 24×24 (quad tab) — left side, tab not in folder */
|
||||
stash24: {
|
||||
region: { x: 23, y: 169, width: 840, height: 840 },
|
||||
cols: 24,
|
||||
rows: 24,
|
||||
},
|
||||
/** Personal stash 24×24 (quad tab) — left side, tab in folder */
|
||||
stash24_folder: {
|
||||
region: { x: 23, y: 216, width: 840, height: 840 },
|
||||
cols: 24,
|
||||
rows: 24,
|
||||
},
|
||||
/** Seller's public stash — always 12×12 */
|
||||
seller: {
|
||||
region: { x: 416, y: 299, width: 840, height: 840 },
|
||||
cols: 12,
|
||||
rows: 12,
|
||||
},
|
||||
/** NPC shop — 12×12 */
|
||||
shop: {
|
||||
region: { x: 23, y: 216, width: 840, height: 840 },
|
||||
cols: 12,
|
||||
rows: 12,
|
||||
},
|
||||
/** NPC vendor inventory — 12×12 */
|
||||
vendor: {
|
||||
region: { x: 416, y: 369, width: 840, height: 840 },
|
||||
cols: 12,
|
||||
rows: 12,
|
||||
},
|
||||
};
|
||||
|
||||
// Backward-compat exports
|
||||
export const INVENTORY = GRID_LAYOUTS.inventory;
|
||||
export const STASH_12x12 = GRID_LAYOUTS.stash12;
|
||||
export const STASH_24x24 = GRID_LAYOUTS.stash24;
|
||||
export const SELLER_12x12 = GRID_LAYOUTS.seller;
|
||||
|
||||
// ── GridReader ──────────────────────────────────────────────────────────────
|
||||
|
||||
export class GridReader {
|
||||
constructor(private daemon: OcrDaemon) {}
|
||||
|
||||
/**
|
||||
* Scan a named grid layout for occupied cells.
|
||||
*/
|
||||
async scan(layoutName: string, threshold?: number, targetRow?: number, targetCol?: number): Promise<ScanResult> {
|
||||
const layout = GRID_LAYOUTS[layoutName];
|
||||
if (!layout) throw new Error(`Unknown grid layout: ${layoutName}`);
|
||||
|
||||
const t = performance.now();
|
||||
const { occupied, items, matches } = await this.getOccupiedCells(layout, threshold, targetRow, targetCol);
|
||||
|
||||
const ms = (performance.now() - t).toFixed(0);
|
||||
logger.info(
|
||||
{ layoutName, cols: layout.cols, rows: layout.rows, occupied: occupied.length, items: items.length, matches: matches?.length, ms },
|
||||
'Grid scan complete',
|
||||
);
|
||||
|
||||
return { layout, occupied, items, matches };
|
||||
}
|
||||
|
||||
/** Get the screen-space center of a grid cell */
|
||||
getCellCenter(layout: GridLayout, row: number, col: number): { x: number; y: number } {
|
||||
const cellW = layout.region.width / layout.cols;
|
||||
const cellH = layout.region.height / layout.rows;
|
||||
return {
|
||||
x: Math.round(layout.region.x + col * cellW + cellW / 2),
|
||||
y: Math.round(layout.region.y + row * cellH + cellH / 2),
|
||||
};
|
||||
}
|
||||
|
||||
/** Scan the grid and return which cells are occupied and detected items */
|
||||
async getOccupiedCells(layout: GridLayout, threshold?: number, targetRow?: number, targetCol?: number): Promise<{ occupied: CellCoord[]; items: GridItem[]; matches?: GridMatch[] }> {
|
||||
const t = performance.now();
|
||||
const result = await this.daemon.gridScan(
|
||||
layout.region,
|
||||
layout.cols,
|
||||
layout.rows,
|
||||
threshold,
|
||||
targetRow,
|
||||
targetCol,
|
||||
);
|
||||
|
||||
const occupied: CellCoord[] = [];
|
||||
for (let row = 0; row < result.cells.length; row++) {
|
||||
for (let col = 0; col < result.cells[row].length; col++) {
|
||||
if (result.cells[row][col]) {
|
||||
const center = this.getCellCenter(layout, row, col);
|
||||
occupied.push({ row, col, x: center.x, y: center.y });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ms = (performance.now() - t).toFixed(0);
|
||||
logger.info(
|
||||
{ layout: `${layout.cols}x${layout.rows}`, occupied: occupied.length, items: result.items.length, matches: result.matches?.length, ms },
|
||||
'Grid scan complete',
|
||||
);
|
||||
return { occupied, items: result.items, matches: result.matches };
|
||||
}
|
||||
|
||||
/** Get all cell centers in the grid */
|
||||
getAllCells(layout: GridLayout): CellCoord[] {
|
||||
const cells: CellCoord[] = [];
|
||||
for (let row = 0; row < layout.rows; row++) {
|
||||
for (let col = 0; col < layout.cols; col++) {
|
||||
const center = this.getCellCenter(layout, row, col);
|
||||
cells.push({ row, col, x: center.x, y: center.y });
|
||||
}
|
||||
}
|
||||
return cells;
|
||||
}
|
||||
}
|
||||
342
src-old/game/InputSender.ts
Normal file
342
src-old/game/InputSender.ts
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
import koffi from 'koffi';
|
||||
import { sleep, randomDelay } from '../util/sleep.js';
|
||||
|
||||
// Win32 POINT struct for GetCursorPos
|
||||
const POINT = koffi.struct('POINT', { x: 'int32', y: 'int32' });
|
||||
|
||||
// Win32 INPUT struct on x64 is 40 bytes:
|
||||
// type (4) + pad (4) + union (32)
|
||||
// MOUSEINPUT is 32 bytes (the largest union member)
|
||||
// KEYBDINPUT is 24 bytes, so needs 8 bytes trailing pad in the union
|
||||
//
|
||||
// We define flat structs that match the exact memory layout,
|
||||
// then cast with koffi.as() when calling SendInput.
|
||||
|
||||
const INPUT_KEYBOARD = koffi.struct('INPUT_KEYBOARD', {
|
||||
type: 'uint32', // offset 0
|
||||
_pad0: 'uint32', // offset 4 (alignment for union at offset 8)
|
||||
wVk: 'uint16', // offset 8
|
||||
wScan: 'uint16', // offset 10
|
||||
dwFlags: 'uint32', // offset 12
|
||||
time: 'uint32', // offset 16
|
||||
_pad1: 'uint32', // offset 20 (alignment for dwExtraInfo)
|
||||
dwExtraInfo: 'uint64', // offset 24
|
||||
_pad2: koffi.array('uint8', 8), // offset 32, pad to 40 bytes total
|
||||
});
|
||||
|
||||
const INPUT_MOUSE = koffi.struct('INPUT_MOUSE', {
|
||||
type: 'uint32', // offset 0
|
||||
_pad0: 'uint32', // offset 4 (alignment for union at offset 8)
|
||||
dx: 'int32', // offset 8
|
||||
dy: 'int32', // offset 12
|
||||
mouseData: 'uint32', // offset 16
|
||||
dwFlags: 'uint32', // offset 20
|
||||
time: 'uint32', // offset 24
|
||||
_pad1: 'uint32', // offset 28 (alignment for dwExtraInfo)
|
||||
dwExtraInfo: 'uint64', // offset 32
|
||||
});
|
||||
// INPUT_MOUSE is already 40 bytes, no trailing pad needed
|
||||
|
||||
const user32 = koffi.load('user32.dll');
|
||||
|
||||
const SendInput = user32.func('SendInput', 'uint32', ['uint32', 'void *', 'int32']);
|
||||
const MapVirtualKeyW = user32.func('MapVirtualKeyW', 'uint32', ['uint32', 'uint32']);
|
||||
const GetSystemMetrics = user32.func('GetSystemMetrics', 'int32', ['int32']);
|
||||
const GetCursorPos = user32.func('GetCursorPos', 'int32', ['_Out_ POINT *']);
|
||||
|
||||
// Constants
|
||||
const INPUT_MOUSE_TYPE = 0;
|
||||
const INPUT_KEYBOARD_TYPE = 1;
|
||||
const KEYEVENTF_SCANCODE = 0x0008;
|
||||
const KEYEVENTF_KEYUP = 0x0002;
|
||||
const KEYEVENTF_UNICODE = 0x0004;
|
||||
|
||||
// Mouse flags
|
||||
const MOUSEEVENTF_MOVE = 0x0001;
|
||||
const MOUSEEVENTF_LEFTDOWN = 0x0002;
|
||||
const MOUSEEVENTF_LEFTUP = 0x0004;
|
||||
const MOUSEEVENTF_RIGHTDOWN = 0x0008;
|
||||
const MOUSEEVENTF_RIGHTUP = 0x0010;
|
||||
const MOUSEEVENTF_ABSOLUTE = 0x8000;
|
||||
|
||||
// System metrics
|
||||
const SM_CXSCREEN = 0;
|
||||
const SM_CYSCREEN = 1;
|
||||
|
||||
// Virtual key codes
|
||||
export const VK = {
|
||||
RETURN: 0x0d,
|
||||
CONTROL: 0x11,
|
||||
MENU: 0x12, // Alt
|
||||
SHIFT: 0x10,
|
||||
ESCAPE: 0x1b,
|
||||
TAB: 0x09,
|
||||
SPACE: 0x20,
|
||||
DELETE: 0x2e,
|
||||
BACK: 0x08,
|
||||
V: 0x56,
|
||||
A: 0x41,
|
||||
C: 0x43,
|
||||
I: 0x49,
|
||||
} as const;
|
||||
|
||||
// Size to pass to SendInput (must be sizeof(INPUT) = 40 on x64)
|
||||
const INPUT_SIZE = koffi.sizeof(INPUT_MOUSE); // 40
|
||||
|
||||
// Bézier curve helpers for natural mouse movement
|
||||
|
||||
function easeInOutQuad(t: number): number {
|
||||
return t < 0.5 ? 2 * t * t : 1 - (-2 * t + 2) ** 2 / 2;
|
||||
}
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
function cubicBezier(t: number, p0: Point, p1: Point, p2: Point, p3: Point): Point {
|
||||
const u = 1 - t;
|
||||
const u2 = u * u;
|
||||
const u3 = u2 * u;
|
||||
const t2 = t * t;
|
||||
const t3 = t2 * t;
|
||||
return {
|
||||
x: u3 * p0.x + 3 * u2 * t * p1.x + 3 * u * t2 * p2.x + t3 * p3.x,
|
||||
y: u3 * p0.y + 3 * u2 * t * p1.y + 3 * u * t2 * p2.y + t3 * p3.y,
|
||||
};
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
export class InputSender {
|
||||
private screenWidth: number;
|
||||
private screenHeight: number;
|
||||
|
||||
constructor() {
|
||||
this.screenWidth = GetSystemMetrics(SM_CXSCREEN);
|
||||
this.screenHeight = GetSystemMetrics(SM_CYSCREEN);
|
||||
}
|
||||
|
||||
async pressKey(vkCode: number): Promise<void> {
|
||||
const scanCode = MapVirtualKeyW(vkCode, 0); // MAPVK_VK_TO_VSC
|
||||
this.sendScanKeyDown(scanCode);
|
||||
await randomDelay(30, 50);
|
||||
this.sendScanKeyUp(scanCode);
|
||||
await randomDelay(20, 40);
|
||||
}
|
||||
|
||||
async keyDown(vkCode: number): Promise<void> {
|
||||
const scanCode = MapVirtualKeyW(vkCode, 0);
|
||||
this.sendScanKeyDown(scanCode);
|
||||
await randomDelay(15, 30);
|
||||
}
|
||||
|
||||
async keyUp(vkCode: number): Promise<void> {
|
||||
const scanCode = MapVirtualKeyW(vkCode, 0);
|
||||
this.sendScanKeyUp(scanCode);
|
||||
await randomDelay(15, 30);
|
||||
}
|
||||
|
||||
async typeText(text: string): Promise<void> {
|
||||
for (const char of text) {
|
||||
this.sendUnicodeChar(char);
|
||||
await randomDelay(20, 50);
|
||||
}
|
||||
}
|
||||
|
||||
async paste(): Promise<void> {
|
||||
await this.keyDown(VK.CONTROL);
|
||||
await sleep(30);
|
||||
await this.pressKey(VK.V);
|
||||
await this.keyUp(VK.CONTROL);
|
||||
await sleep(50);
|
||||
}
|
||||
|
||||
async selectAll(): Promise<void> {
|
||||
await this.keyDown(VK.CONTROL);
|
||||
await sleep(30);
|
||||
await this.pressKey(VK.A);
|
||||
await this.keyUp(VK.CONTROL);
|
||||
await sleep(50);
|
||||
}
|
||||
|
||||
private getCursorPos(): Point {
|
||||
const pt = { x: 0, y: 0 };
|
||||
GetCursorPos(pt);
|
||||
return pt;
|
||||
}
|
||||
|
||||
private moveMouseRaw(x: number, y: number): void {
|
||||
const normalizedX = Math.round((x * 65535) / this.screenWidth);
|
||||
const normalizedY = Math.round((y * 65535) / this.screenHeight);
|
||||
this.sendMouseInput(normalizedX, normalizedY, 0, MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE);
|
||||
}
|
||||
|
||||
async moveMouse(x: number, y: number): Promise<void> {
|
||||
const start = this.getCursorPos();
|
||||
const end: Point = { x, y };
|
||||
const dx = end.x - start.x;
|
||||
const dy = end.y - start.y;
|
||||
const distance = Math.sqrt(dx * dx + dy * dy);
|
||||
|
||||
// Short distance: just teleport
|
||||
if (distance < 10) {
|
||||
this.moveMouseRaw(x, y);
|
||||
await randomDelay(10, 20);
|
||||
return;
|
||||
}
|
||||
|
||||
// Generate 2 random control points offset from the straight line
|
||||
const perpX = -dy / distance;
|
||||
const perpY = dx / distance;
|
||||
const spread = distance * 0.3;
|
||||
|
||||
const cp1: Point = {
|
||||
x: start.x + dx * 0.25 + perpX * (Math.random() - 0.5) * spread,
|
||||
y: start.y + dy * 0.25 + perpY * (Math.random() - 0.5) * spread,
|
||||
};
|
||||
const cp2: Point = {
|
||||
x: start.x + dx * 0.75 + perpX * (Math.random() - 0.5) * spread,
|
||||
y: start.y + dy * 0.75 + perpY * (Math.random() - 0.5) * spread,
|
||||
};
|
||||
|
||||
const steps = clamp(Math.round(distance / 30), 8, 20);
|
||||
|
||||
for (let i = 1; i <= steps; i++) {
|
||||
const rawT = i / steps;
|
||||
const t = easeInOutQuad(rawT);
|
||||
const pt = cubicBezier(t, start, cp1, cp2, end);
|
||||
|
||||
// Add ±1px jitter except on the last step
|
||||
const jitterX = i < steps ? Math.round((Math.random() - 0.5) * 2) : 0;
|
||||
const jitterY = i < steps ? Math.round((Math.random() - 0.5) * 2) : 0;
|
||||
|
||||
this.moveMouseRaw(Math.round(pt.x) + jitterX, Math.round(pt.y) + jitterY);
|
||||
await sleep(1 + Math.random() * 2); // 1-3ms between steps
|
||||
}
|
||||
|
||||
// Final exact landing
|
||||
this.moveMouseRaw(x, y);
|
||||
await randomDelay(5, 15);
|
||||
}
|
||||
|
||||
moveMouseInstant(x: number, y: number): void {
|
||||
this.moveMouseRaw(x, y);
|
||||
}
|
||||
|
||||
/** Quick Bézier move — ~10-15ms, 5 steps, no jitter. Fast but not a raw teleport. */
|
||||
async moveMouseFast(x: number, y: number): Promise<void> {
|
||||
const start = this.getCursorPos();
|
||||
const end: Point = { x, y };
|
||||
const dx = end.x - start.x;
|
||||
const dy = end.y - start.y;
|
||||
const distance = Math.sqrt(dx * dx + dy * dy);
|
||||
|
||||
if (distance < 10) {
|
||||
this.moveMouseRaw(x, y);
|
||||
return;
|
||||
}
|
||||
|
||||
const perpX = -dy / distance;
|
||||
const perpY = dx / distance;
|
||||
const spread = distance * 0.15;
|
||||
|
||||
const cp1: Point = {
|
||||
x: start.x + dx * 0.3 + perpX * (Math.random() - 0.5) * spread,
|
||||
y: start.y + dy * 0.3 + perpY * (Math.random() - 0.5) * spread,
|
||||
};
|
||||
const cp2: Point = {
|
||||
x: start.x + dx * 0.7 + perpX * (Math.random() - 0.5) * spread,
|
||||
y: start.y + dy * 0.7 + perpY * (Math.random() - 0.5) * spread,
|
||||
};
|
||||
|
||||
const steps = 5;
|
||||
for (let i = 1; i <= steps; i++) {
|
||||
const t = easeInOutQuad(i / steps);
|
||||
const pt = cubicBezier(t, start, cp1, cp2, end);
|
||||
this.moveMouseRaw(Math.round(pt.x), Math.round(pt.y));
|
||||
await sleep(2);
|
||||
}
|
||||
this.moveMouseRaw(x, y);
|
||||
}
|
||||
|
||||
async leftClick(x: number, y: number): Promise<void> {
|
||||
await this.moveMouse(x, y);
|
||||
await randomDelay(20, 50);
|
||||
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_LEFTDOWN);
|
||||
await randomDelay(15, 40);
|
||||
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_LEFTUP);
|
||||
await randomDelay(15, 30);
|
||||
}
|
||||
|
||||
async rightClick(x: number, y: number): Promise<void> {
|
||||
await this.moveMouse(x, y);
|
||||
await randomDelay(20, 50);
|
||||
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_RIGHTDOWN);
|
||||
await randomDelay(15, 40);
|
||||
this.sendMouseInput(0, 0, 0, MOUSEEVENTF_RIGHTUP);
|
||||
await randomDelay(15, 30);
|
||||
}
|
||||
|
||||
async ctrlRightClick(x: number, y: number): Promise<void> {
|
||||
await this.keyDown(VK.CONTROL);
|
||||
await randomDelay(30, 60);
|
||||
await this.rightClick(x, y);
|
||||
await this.keyUp(VK.CONTROL);
|
||||
await randomDelay(30, 60);
|
||||
}
|
||||
|
||||
async ctrlLeftClick(x: number, y: number): Promise<void> {
|
||||
await this.keyDown(VK.CONTROL);
|
||||
await randomDelay(30, 60);
|
||||
await this.leftClick(x, y);
|
||||
await this.keyUp(VK.CONTROL);
|
||||
await randomDelay(30, 60);
|
||||
}
|
||||
|
||||
private sendMouseInput(dx: number, dy: number, mouseData: number, flags: number): void {
|
||||
const input = {
|
||||
type: INPUT_MOUSE_TYPE,
|
||||
_pad0: 0,
|
||||
dx,
|
||||
dy,
|
||||
mouseData,
|
||||
dwFlags: flags,
|
||||
time: 0,
|
||||
_pad1: 0,
|
||||
dwExtraInfo: 0,
|
||||
};
|
||||
SendInput(1, koffi.as(input, 'INPUT_MOUSE *'), INPUT_SIZE);
|
||||
}
|
||||
|
||||
private sendKeyInput(wVk: number, wScan: number, flags: number): void {
|
||||
const input = {
|
||||
type: INPUT_KEYBOARD_TYPE,
|
||||
_pad0: 0,
|
||||
wVk,
|
||||
wScan,
|
||||
dwFlags: flags,
|
||||
time: 0,
|
||||
_pad1: 0,
|
||||
dwExtraInfo: 0,
|
||||
_pad2: [0, 0, 0, 0, 0, 0, 0, 0],
|
||||
};
|
||||
SendInput(1, koffi.as(input, 'INPUT_KEYBOARD *'), INPUT_SIZE);
|
||||
}
|
||||
|
||||
private sendScanKeyDown(scanCode: number): void {
|
||||
this.sendKeyInput(0, scanCode, KEYEVENTF_SCANCODE);
|
||||
}
|
||||
|
||||
private sendScanKeyUp(scanCode: number): void {
|
||||
this.sendKeyInput(0, scanCode, KEYEVENTF_SCANCODE | KEYEVENTF_KEYUP);
|
||||
}
|
||||
|
||||
private sendUnicodeChar(char: string): void {
|
||||
const code = char.charCodeAt(0);
|
||||
this.sendKeyInput(0, code, KEYEVENTF_UNICODE);
|
||||
this.sendKeyInput(0, code, KEYEVENTF_UNICODE | KEYEVENTF_KEYUP);
|
||||
}
|
||||
}
|
||||
464
src-old/game/OcrDaemon.ts
Normal file
464
src-old/game/OcrDaemon.ts
Normal file
|
|
@ -0,0 +1,464 @@
|
|||
import { spawn, type ChildProcess } from 'child_process';
|
||||
import { join } from 'path';
|
||||
import { logger } from '../util/logger.js';
|
||||
import type { Region } from '../types.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface OcrWord {
|
||||
text: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface OcrLine {
|
||||
text: string;
|
||||
words: OcrWord[];
|
||||
}
|
||||
|
||||
export interface OcrResponse {
|
||||
ok: true;
|
||||
text: string;
|
||||
lines: OcrLine[];
|
||||
}
|
||||
|
||||
export interface GridItem {
|
||||
row: number;
|
||||
col: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
export interface GridMatch {
|
||||
row: number;
|
||||
col: number;
|
||||
similarity: number;
|
||||
}
|
||||
|
||||
export interface GridScanResult {
|
||||
cells: boolean[][];
|
||||
items: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
}
|
||||
|
||||
export interface DiffOcrResponse {
|
||||
text: string;
|
||||
lines: OcrLine[];
|
||||
region?: Region;
|
||||
}
|
||||
|
||||
export interface DetectGridResult {
|
||||
detected: boolean;
|
||||
region?: Region;
|
||||
cols?: number;
|
||||
rows?: number;
|
||||
cellWidth?: number;
|
||||
cellHeight?: number;
|
||||
}
|
||||
|
||||
export interface TemplateMatchResult {
|
||||
found: boolean;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
export type OcrEngine = 'tesseract' | 'easyocr' | 'paddleocr';
|
||||
|
||||
export type OcrPreprocess = 'none' | 'bgsub' | 'tophat';
|
||||
|
||||
export interface DiffCropParams {
|
||||
diffThresh?: number;
|
||||
rowThreshDiv?: number;
|
||||
colThreshDiv?: number;
|
||||
maxGap?: number;
|
||||
trimCutoff?: number;
|
||||
ocrPad?: number;
|
||||
}
|
||||
|
||||
export interface OcrParams {
|
||||
kernelSize?: number;
|
||||
upscale?: number;
|
||||
useBackgroundSub?: boolean;
|
||||
dimPercentile?: number;
|
||||
textThresh?: number;
|
||||
softThreshold?: boolean;
|
||||
usePerLineOcr?: boolean;
|
||||
lineGapTolerance?: number;
|
||||
linePadY?: number;
|
||||
psm?: number;
|
||||
mergeGap?: number;
|
||||
linkThreshold?: number;
|
||||
textThreshold?: number;
|
||||
lowText?: number;
|
||||
widthThs?: number;
|
||||
paragraph?: boolean;
|
||||
}
|
||||
|
||||
export interface DiffOcrParams {
|
||||
crop?: DiffCropParams;
|
||||
ocr?: OcrParams;
|
||||
}
|
||||
|
||||
export type TooltipMethod = 'diff' | 'edge';
|
||||
|
||||
export interface EdgeCropParams {
|
||||
cannyLow?: number;
|
||||
cannyHigh?: number;
|
||||
minLineLength?: number;
|
||||
roiSize?: number;
|
||||
densityThreshold?: number;
|
||||
ocrPad?: number;
|
||||
}
|
||||
|
||||
export interface EdgeOcrParams {
|
||||
crop?: EdgeCropParams;
|
||||
ocr?: OcrParams;
|
||||
}
|
||||
|
||||
interface DaemonRequest {
|
||||
cmd: string;
|
||||
region?: Region;
|
||||
path?: string;
|
||||
cols?: number;
|
||||
rows?: number;
|
||||
threshold?: number;
|
||||
minCellSize?: number;
|
||||
maxCellSize?: number;
|
||||
engine?: string;
|
||||
preprocess?: string;
|
||||
params?: DiffOcrParams;
|
||||
edgeParams?: EdgeOcrParams;
|
||||
cursorX?: number;
|
||||
cursorY?: number;
|
||||
}
|
||||
|
||||
interface DaemonResponse {
|
||||
ok: boolean;
|
||||
ready?: boolean;
|
||||
text?: string;
|
||||
lines?: OcrLine[];
|
||||
image?: string;
|
||||
cells?: boolean[][];
|
||||
items?: GridItem[];
|
||||
matches?: GridMatch[];
|
||||
detected?: boolean;
|
||||
region?: Region;
|
||||
cols?: number;
|
||||
rows?: number;
|
||||
cellWidth?: number;
|
||||
cellHeight?: number;
|
||||
found?: boolean;
|
||||
x?: number;
|
||||
y?: number;
|
||||
width?: number;
|
||||
height?: number;
|
||||
confidence?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
// ── OcrDaemon ───────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_EXE = join(
|
||||
'tools', 'OcrDaemon', 'bin', 'Release',
|
||||
'net8.0-windows10.0.19041.0', 'OcrDaemon.exe',
|
||||
);
|
||||
|
||||
const REQUEST_TIMEOUT = 5_000;
|
||||
const CAPTURE_TIMEOUT = 10_000;
|
||||
|
||||
export class OcrDaemon {
|
||||
private proc: ChildProcess | null = null;
|
||||
private exePath: string;
|
||||
private readyResolve: ((value: void) => void) | null = null;
|
||||
private readyReject: ((err: Error) => void) | null = null;
|
||||
private pendingResolve: ((resp: DaemonResponse) => void) | null = null;
|
||||
private pendingReject: ((err: Error) => void) | null = null;
|
||||
private queue: Array<{ request: DaemonRequest; resolve: (resp: DaemonResponse) => void; reject: (err: Error) => void }> = [];
|
||||
private processing = false;
|
||||
private buffer = '';
|
||||
private stopped = false;
|
||||
|
||||
constructor(exePath?: string) {
|
||||
this.exePath = exePath ?? DEFAULT_EXE;
|
||||
}
|
||||
|
||||
// ── Public API ──────────────────────────────────────────────────────────
|
||||
|
||||
async ocr(region?: Region, engine?: OcrEngine, preprocess?: OcrPreprocess): Promise<OcrResponse> {
|
||||
const req: DaemonRequest = { cmd: 'ocr' };
|
||||
if (region) req.region = region;
|
||||
if (engine && engine !== 'tesseract') req.engine = engine;
|
||||
if (preprocess && preprocess !== 'none') req.preprocess = preprocess;
|
||||
// Python engines need longer timeout for first model load + download
|
||||
const timeout = (engine && engine !== 'tesseract') ? 120_000 : CAPTURE_TIMEOUT;
|
||||
const resp = await this.sendWithRetry(req, timeout);
|
||||
return {
|
||||
ok: true,
|
||||
text: resp.text ?? '',
|
||||
lines: resp.lines ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
async captureBuffer(region?: Region): Promise<Buffer> {
|
||||
const req: DaemonRequest = { cmd: 'capture' };
|
||||
if (region) req.region = region;
|
||||
const resp = await this.sendWithRetry(req, CAPTURE_TIMEOUT);
|
||||
return Buffer.from(resp.image!, 'base64');
|
||||
}
|
||||
|
||||
async gridScan(region: Region, cols: number, rows: number, threshold?: number, targetRow?: number, targetCol?: number): Promise<GridScanResult> {
|
||||
const req: DaemonRequest = { cmd: 'grid', region, cols, rows };
|
||||
if (threshold) req.threshold = threshold;
|
||||
if (targetRow != null && targetRow >= 0) (req as any).targetRow = targetRow;
|
||||
if (targetCol != null && targetCol >= 0) (req as any).targetCol = targetCol;
|
||||
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
|
||||
return { cells: resp.cells ?? [], items: resp.items ?? [], matches: resp.matches ?? undefined };
|
||||
}
|
||||
|
||||
async detectGrid(region: Region, minCellSize?: number, maxCellSize?: number): Promise<DetectGridResult> {
|
||||
const req: DaemonRequest = { cmd: 'detect-grid', region };
|
||||
if (minCellSize) req.minCellSize = minCellSize;
|
||||
if (maxCellSize) req.maxCellSize = maxCellSize;
|
||||
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
|
||||
return {
|
||||
detected: resp.detected ?? false,
|
||||
region: resp.region,
|
||||
cols: resp.cols,
|
||||
rows: resp.rows,
|
||||
cellWidth: resp.cellWidth,
|
||||
cellHeight: resp.cellHeight,
|
||||
};
|
||||
}
|
||||
|
||||
async snapshot(): Promise<void> {
|
||||
await this.sendWithRetry({ cmd: 'snapshot' }, REQUEST_TIMEOUT);
|
||||
}
|
||||
|
||||
async diffOcr(savePath?: string, region?: Region, engine?: OcrEngine, preprocess?: OcrPreprocess, params?: DiffOcrParams): Promise<DiffOcrResponse> {
|
||||
const req: DaemonRequest = { cmd: 'diff-ocr' };
|
||||
if (savePath) req.path = savePath;
|
||||
if (region) req.region = region;
|
||||
if (engine && engine !== 'tesseract') req.engine = engine;
|
||||
if (preprocess) req.preprocess = preprocess;
|
||||
if (params && Object.keys(params).length > 0) req.params = params;
|
||||
const timeout = (engine && engine !== 'tesseract') ? 120_000 : CAPTURE_TIMEOUT;
|
||||
const resp = await this.sendWithRetry(req, timeout);
|
||||
return {
|
||||
text: resp.text ?? '',
|
||||
lines: resp.lines ?? [],
|
||||
region: resp.region,
|
||||
};
|
||||
}
|
||||
|
||||
async edgeOcr(savePath?: string, region?: Region, engine?: OcrEngine, preprocess?: OcrPreprocess, edgeParams?: EdgeOcrParams, cursorX?: number, cursorY?: number): Promise<DiffOcrResponse> {
|
||||
const req: DaemonRequest = { cmd: 'edge-ocr' };
|
||||
if (savePath) req.path = savePath;
|
||||
if (region) req.region = region;
|
||||
if (engine && engine !== 'tesseract') req.engine = engine;
|
||||
if (preprocess) req.preprocess = preprocess;
|
||||
if (edgeParams && Object.keys(edgeParams).length > 0) req.edgeParams = edgeParams;
|
||||
if (cursorX != null) req.cursorX = cursorX;
|
||||
if (cursorY != null) req.cursorY = cursorY;
|
||||
const timeout = (engine && engine !== 'tesseract') ? 120_000 : CAPTURE_TIMEOUT;
|
||||
const resp = await this.sendWithRetry(req, timeout);
|
||||
return {
|
||||
text: resp.text ?? '',
|
||||
lines: resp.lines ?? [],
|
||||
region: resp.region,
|
||||
};
|
||||
}
|
||||
|
||||
async saveScreenshot(path: string, region?: Region): Promise<void> {
|
||||
const req: DaemonRequest = { cmd: 'screenshot', path };
|
||||
if (region) req.region = region;
|
||||
await this.sendWithRetry(req, REQUEST_TIMEOUT);
|
||||
}
|
||||
|
||||
async templateMatch(templatePath: string, region?: Region): Promise<TemplateMatchResult | null> {
|
||||
const req: DaemonRequest = { cmd: 'match-template', path: templatePath };
|
||||
if (region) req.region = region;
|
||||
const resp = await this.sendWithRetry(req, REQUEST_TIMEOUT);
|
||||
if (!resp.found) return null;
|
||||
return {
|
||||
found: true,
|
||||
x: resp.x!,
|
||||
y: resp.y!,
|
||||
width: resp.width!,
|
||||
height: resp.height!,
|
||||
confidence: resp.confidence!,
|
||||
};
|
||||
}
|
||||
|
||||
/** Eagerly spawn the daemon process so it's ready for the first real request. */
|
||||
async warmup(): Promise<void> {
|
||||
await this.ensureRunning();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.stopped = true;
|
||||
if (this.proc) {
|
||||
const p = this.proc;
|
||||
this.proc = null;
|
||||
p.stdin?.end();
|
||||
p.kill();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Internal ────────────────────────────────────────────────────────────
|
||||
|
||||
private async ensureRunning(): Promise<void> {
|
||||
if (this.proc && this.proc.exitCode === null) return;
|
||||
|
||||
this.proc = null;
|
||||
this.buffer = '';
|
||||
|
||||
logger.info({ exe: this.exePath }, 'Spawning OCR daemon');
|
||||
|
||||
const proc = spawn(this.exePath, [], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
windowsHide: true,
|
||||
});
|
||||
|
||||
this.proc = proc;
|
||||
|
||||
proc.stderr?.on('data', (data: Buffer) => {
|
||||
logger.warn({ daemon: data.toString().trim() }, 'OcrDaemon stderr');
|
||||
});
|
||||
|
||||
proc.on('exit', (code) => {
|
||||
logger.warn({ code }, 'OcrDaemon exited');
|
||||
if (this.pendingReject) {
|
||||
this.pendingReject(new Error(`Daemon exited with code ${code}`));
|
||||
this.pendingResolve = null;
|
||||
this.pendingReject = null;
|
||||
}
|
||||
});
|
||||
|
||||
proc.stdout!.on('data', (data: Buffer) => {
|
||||
this.buffer += data.toString();
|
||||
this.processBuffer();
|
||||
});
|
||||
|
||||
// Wait for ready signal
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
this.readyResolve = resolve;
|
||||
this.readyReject = reject;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
this.readyReject = null;
|
||||
this.readyResolve = null;
|
||||
reject(new Error('Daemon did not become ready within 10s'));
|
||||
}, 10_000);
|
||||
|
||||
// Store so we can clear on resolve
|
||||
(this as any)._readyTimeout = timeout;
|
||||
});
|
||||
|
||||
logger.info('OCR daemon ready');
|
||||
}
|
||||
|
||||
private processBuffer(): void {
|
||||
let newlineIdx: number;
|
||||
while ((newlineIdx = this.buffer.indexOf('\n')) !== -1) {
|
||||
const line = this.buffer.slice(0, newlineIdx).trim();
|
||||
this.buffer = this.buffer.slice(newlineIdx + 1);
|
||||
|
||||
if (!line) continue;
|
||||
|
||||
let parsed: DaemonResponse;
|
||||
try {
|
||||
parsed = JSON.parse(line);
|
||||
} catch {
|
||||
logger.warn({ line }, 'Failed to parse daemon response');
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle ready signal
|
||||
if (parsed.ready && this.readyResolve) {
|
||||
clearTimeout((this as any)._readyTimeout);
|
||||
const resolve = this.readyResolve;
|
||||
this.readyResolve = null;
|
||||
this.readyReject = null;
|
||||
resolve();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle normal response
|
||||
if (this.pendingResolve) {
|
||||
const resolve = this.pendingResolve;
|
||||
this.pendingResolve = null;
|
||||
this.pendingReject = null;
|
||||
resolve(parsed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async send(request: DaemonRequest, timeout: number): Promise<DaemonResponse> {
|
||||
await this.ensureRunning();
|
||||
|
||||
return new Promise<DaemonResponse>((resolve, reject) => {
|
||||
this.queue.push({ request, resolve, reject });
|
||||
this.drainQueue(timeout);
|
||||
});
|
||||
}
|
||||
|
||||
private drainQueue(timeout: number): void {
|
||||
if (this.processing || this.queue.length === 0) return;
|
||||
this.processing = true;
|
||||
|
||||
const { request, resolve, reject } = this.queue.shift()!;
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
this.pendingResolve = null;
|
||||
this.pendingReject = null;
|
||||
this.processing = false;
|
||||
reject(new Error(`Daemon request timed out after ${timeout}ms`));
|
||||
this.drainQueue(timeout);
|
||||
}, timeout);
|
||||
|
||||
this.pendingResolve = (resp) => {
|
||||
clearTimeout(timer);
|
||||
this.processing = false;
|
||||
resolve(resp);
|
||||
this.drainQueue(timeout);
|
||||
};
|
||||
|
||||
this.pendingReject = (err) => {
|
||||
clearTimeout(timer);
|
||||
this.processing = false;
|
||||
reject(err);
|
||||
this.drainQueue(timeout);
|
||||
};
|
||||
|
||||
const json = JSON.stringify(request) + '\n';
|
||||
this.proc!.stdin!.write(json);
|
||||
}
|
||||
|
||||
private async sendWithRetry(request: DaemonRequest, timeout: number): Promise<DaemonResponse> {
|
||||
try {
|
||||
const resp = await this.send(request, timeout);
|
||||
if (!resp.ok) throw new Error(resp.error ?? 'Daemon returned error');
|
||||
return resp;
|
||||
} catch (err) {
|
||||
if (this.stopped) throw err;
|
||||
|
||||
// Kill and retry once
|
||||
logger.warn({ err, cmd: request.cmd }, 'Daemon request failed, restarting');
|
||||
if (this.proc) {
|
||||
const p = this.proc;
|
||||
this.proc = null;
|
||||
p.stdin?.end();
|
||||
p.kill();
|
||||
}
|
||||
|
||||
const resp = await this.send(request, timeout);
|
||||
if (!resp.ok) throw new Error(resp.error ?? 'Daemon returned error on retry');
|
||||
return resp;
|
||||
}
|
||||
}
|
||||
}
|
||||
297
src-old/game/ScreenReader.ts
Normal file
297
src-old/game/ScreenReader.ts
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
import { mkdir } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { logger } from '../util/logger.js';
|
||||
import { OcrDaemon, type OcrResponse, type OcrEngine, type OcrPreprocess, type DiffOcrParams, type DiffCropParams, type OcrParams, type DiffOcrResponse, type TemplateMatchResult, type TooltipMethod, type EdgeOcrParams } from './OcrDaemon.js';
|
||||
import { GridReader, type GridLayout, type CellCoord } from './GridReader.js';
|
||||
import type { Region } from '../types.js';
|
||||
|
||||
function elapsed(start: number): string {
|
||||
return `${(performance.now() - start).toFixed(0)}ms`;
|
||||
}
|
||||
|
||||
export interface OcrSettings {
|
||||
engine: OcrEngine;
|
||||
screenPreprocess: OcrPreprocess;
|
||||
tooltipPreprocess: OcrPreprocess;
|
||||
tooltipMethod: TooltipMethod;
|
||||
tooltipParams: DiffOcrParams;
|
||||
edgeParams: EdgeOcrParams;
|
||||
saveDebugImages: boolean;
|
||||
}
|
||||
|
||||
export class ScreenReader {
|
||||
private daemon = new OcrDaemon();
|
||||
readonly grid = new GridReader(this.daemon);
|
||||
settings: OcrSettings = {
|
||||
engine: 'easyocr',
|
||||
screenPreprocess: 'none',
|
||||
tooltipPreprocess: 'tophat',
|
||||
tooltipMethod: 'diff',
|
||||
tooltipParams: {
|
||||
crop: { diffThresh: 10 },
|
||||
ocr: { kernelSize: 21 },
|
||||
},
|
||||
edgeParams: {
|
||||
crop: {},
|
||||
ocr: { kernelSize: 21 },
|
||||
},
|
||||
saveDebugImages: true,
|
||||
};
|
||||
|
||||
/**
|
||||
* Eagerly spawn the OCR daemon and warm up the EasyOCR model.
|
||||
* Fire-and-forget a small OCR request so the Python model loads in the background.
|
||||
*/
|
||||
async warmup(): Promise<void> {
|
||||
await this.daemon.warmup();
|
||||
// Fire a small EasyOCR request to trigger Python model load
|
||||
// Use a tiny 1×1 region to minimize work, we only care about loading the model
|
||||
const { engine } = this.settings;
|
||||
if (engine !== 'tesseract') {
|
||||
await this.daemon.ocr({ x: 0, y: 0, width: 100, height: 100 }, engine);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Screenshot capture ──────────────────────────────────────────────
|
||||
|
||||
async captureScreen(): Promise<Buffer> {
|
||||
const t = performance.now();
|
||||
const buf = await this.daemon.captureBuffer();
|
||||
logger.info({ ms: elapsed(t) }, 'captureScreen');
|
||||
return buf;
|
||||
}
|
||||
|
||||
async captureRegion(region: Region): Promise<Buffer> {
|
||||
const t = performance.now();
|
||||
const buf = await this.daemon.captureBuffer(region);
|
||||
logger.info({ ms: elapsed(t) }, 'captureRegion');
|
||||
return buf;
|
||||
}
|
||||
|
||||
// ── OCR helpers ─────────────────────────────────────────────────────
|
||||
|
||||
/** Bigram (Dice) similarity between two strings, 0..1. */
|
||||
private static bigramSimilarity(a: string, b: string): number {
|
||||
if (a.length < 2 || b.length < 2) return a === b ? 1 : 0;
|
||||
const bigramsA = new Map<string, number>();
|
||||
for (let i = 0; i < a.length - 1; i++) {
|
||||
const bg = a.slice(i, i + 2);
|
||||
bigramsA.set(bg, (bigramsA.get(bg) ?? 0) + 1);
|
||||
}
|
||||
let matches = 0;
|
||||
for (let i = 0; i < b.length - 1; i++) {
|
||||
const bg = b.slice(i, i + 2);
|
||||
const count = bigramsA.get(bg);
|
||||
if (count && count > 0) {
|
||||
matches++;
|
||||
bigramsA.set(bg, count - 1);
|
||||
}
|
||||
}
|
||||
return (2 * matches) / (a.length - 1 + b.length - 1);
|
||||
}
|
||||
|
||||
/** Normalize text for fuzzy comparison: lowercase, strip non-alphanumeric, collapse spaces. */
|
||||
private static normalize(s: string): string {
|
||||
return s.toLowerCase().replace(/[^a-z0-9]/g, '');
|
||||
}
|
||||
|
||||
private findWordInOcrResult(
|
||||
result: OcrResponse,
|
||||
needle: string,
|
||||
fuzzy: boolean = false,
|
||||
): { x: number; y: number } | null {
|
||||
const lower = needle.toLowerCase();
|
||||
const FUZZY_THRESHOLD = 0.55;
|
||||
|
||||
// Multi-word: match against the full line text, return center of the line's bounding box
|
||||
if (lower.includes(' ')) {
|
||||
const needleNorm = ScreenReader.normalize(needle);
|
||||
|
||||
for (const line of result.lines) {
|
||||
if (line.words.length === 0) continue;
|
||||
|
||||
const lineText = line.text.toLowerCase();
|
||||
// Exact match
|
||||
if (lineText.includes(lower)) {
|
||||
return this.lineBounds(line);
|
||||
}
|
||||
|
||||
// Fuzzy: normalize line text and check sliding windows
|
||||
if (fuzzy) {
|
||||
const lineNorm = ScreenReader.normalize(line.text);
|
||||
// Check windows of similar length to the needle
|
||||
const windowLen = needleNorm.length;
|
||||
for (let i = 0; i <= lineNorm.length - windowLen + 2; i++) {
|
||||
const window = lineNorm.slice(i, i + windowLen + 2);
|
||||
const sim = ScreenReader.bigramSimilarity(needleNorm, window);
|
||||
if (sim >= FUZZY_THRESHOLD) {
|
||||
logger.info({ needle, matched: line.text, similarity: sim.toFixed(2) }, 'Fuzzy nameplate match');
|
||||
return this.lineBounds(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Single word: match against individual words
|
||||
const needleNorm = ScreenReader.normalize(needle);
|
||||
for (const line of result.lines) {
|
||||
for (const word of line.words) {
|
||||
// Exact match
|
||||
if (word.text.toLowerCase().includes(lower)) {
|
||||
return {
|
||||
x: Math.round(word.x + word.width / 2),
|
||||
y: Math.round(word.y + word.height / 2),
|
||||
};
|
||||
}
|
||||
|
||||
// Fuzzy match
|
||||
if (fuzzy) {
|
||||
const wordNorm = ScreenReader.normalize(word.text);
|
||||
const sim = ScreenReader.bigramSimilarity(needleNorm, wordNorm);
|
||||
if (sim >= FUZZY_THRESHOLD) {
|
||||
logger.info({ needle, matched: word.text, similarity: sim.toFixed(2) }, 'Fuzzy word match');
|
||||
return {
|
||||
x: Math.round(word.x + word.width / 2),
|
||||
y: Math.round(word.y + word.height / 2),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Get center of a line's bounding box from its words. */
|
||||
private lineBounds(line: { words: { x: number; y: number; width: number; height: number }[] }): { x: number; y: number } {
|
||||
const first = line.words[0];
|
||||
const last = line.words[line.words.length - 1];
|
||||
const x1 = first.x;
|
||||
const y1 = first.y;
|
||||
const x2 = last.x + last.width;
|
||||
const y2 = Math.max(...line.words.map(w => w.y + w.height));
|
||||
return {
|
||||
x: Math.round((x1 + x2) / 2),
|
||||
y: Math.round((y1 + y2) / 2),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Full-screen methods ─────────────────────────────────────────────
|
||||
|
||||
async findTextOnScreen(
|
||||
searchText: string,
|
||||
fuzzy: boolean = false,
|
||||
): Promise<{ x: number; y: number } | null> {
|
||||
const t = performance.now();
|
||||
const { engine, screenPreprocess } = this.settings;
|
||||
const pp = screenPreprocess !== 'none' ? screenPreprocess : undefined;
|
||||
const result = await this.daemon.ocr(undefined, engine, pp);
|
||||
const pos = this.findWordInOcrResult(result, searchText, fuzzy);
|
||||
|
||||
if (pos) {
|
||||
logger.info({ searchText, engine, x: pos.x, y: pos.y, totalMs: elapsed(t) }, 'Found text on screen');
|
||||
} else {
|
||||
logger.info({ searchText, engine, totalMs: elapsed(t) }, 'Text not found on screen');
|
||||
}
|
||||
return pos;
|
||||
}
|
||||
|
||||
async readFullScreen(): Promise<string> {
|
||||
const { engine, screenPreprocess } = this.settings;
|
||||
const pp = screenPreprocess !== 'none' ? screenPreprocess : undefined;
|
||||
const result = await this.daemon.ocr(undefined, engine, pp);
|
||||
return result.text;
|
||||
}
|
||||
|
||||
// ── Region methods ──────────────────────────────────────────────────
|
||||
|
||||
async findTextInRegion(
|
||||
region: Region,
|
||||
searchText: string,
|
||||
): Promise<{ x: number; y: number } | null> {
|
||||
const t = performance.now();
|
||||
const { engine, screenPreprocess } = this.settings;
|
||||
const pp = screenPreprocess !== 'none' ? screenPreprocess : undefined;
|
||||
const result = await this.daemon.ocr(region, engine, pp);
|
||||
const pos = this.findWordInOcrResult(result, searchText);
|
||||
|
||||
if (pos) {
|
||||
// Offset back to screen space
|
||||
const screenPos = { x: region.x + pos.x, y: region.y + pos.y };
|
||||
logger.info({ searchText, x: screenPos.x, y: screenPos.y, region, totalMs: elapsed(t) }, 'Found text in region');
|
||||
return screenPos;
|
||||
}
|
||||
|
||||
logger.info({ searchText, region, totalMs: elapsed(t) }, 'Text not found in region');
|
||||
return null;
|
||||
}
|
||||
|
||||
async readRegionText(region: Region): Promise<string> {
|
||||
const { engine, screenPreprocess } = this.settings;
|
||||
const pp = screenPreprocess !== 'none' ? screenPreprocess : undefined;
|
||||
const result = await this.daemon.ocr(region, engine, pp);
|
||||
return result.text;
|
||||
}
|
||||
|
||||
async checkForText(region: Region, searchText: string): Promise<boolean> {
|
||||
const pos = await this.findTextInRegion(region, searchText);
|
||||
return pos !== null;
|
||||
}
|
||||
|
||||
// ── Snapshot / Diff-OCR (for tooltip reading) ──────────────────────
|
||||
|
||||
async snapshot(): Promise<void> {
|
||||
if (this.settings.tooltipMethod === 'edge') return; // no reference frame needed
|
||||
await this.daemon.snapshot();
|
||||
}
|
||||
|
||||
async diffOcr(savePath?: string, region?: Region): Promise<DiffOcrResponse> {
|
||||
const { engine, tooltipPreprocess, tooltipMethod, tooltipParams, edgeParams } = this.settings;
|
||||
const pp = tooltipPreprocess !== 'none' ? tooltipPreprocess : undefined;
|
||||
if (tooltipMethod === 'edge') {
|
||||
return this.daemon.edgeOcr(savePath, region, engine, pp, edgeParams);
|
||||
}
|
||||
return this.daemon.diffOcr(savePath, region, engine, pp, tooltipParams);
|
||||
}
|
||||
|
||||
// ── Template matching ──────────────────────────────────────────────
|
||||
|
||||
async templateMatch(templatePath: string, region?: Region): Promise<TemplateMatchResult | null> {
|
||||
const t = performance.now();
|
||||
const result = await this.daemon.templateMatch(templatePath, region);
|
||||
if (result) {
|
||||
logger.info({ templatePath, x: result.x, y: result.y, confidence: result.confidence.toFixed(3), ms: elapsed(t) }, 'Template match found');
|
||||
} else {
|
||||
logger.info({ templatePath, ms: elapsed(t) }, 'Template match not found');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Save utilities ──────────────────────────────────────────────────
|
||||
|
||||
async saveScreenshot(path: string): Promise<void> {
|
||||
await this.daemon.saveScreenshot(path);
|
||||
logger.info({ path }, 'Screenshot saved');
|
||||
}
|
||||
|
||||
async saveDebugScreenshots(dir: string): Promise<string[]> {
|
||||
await mkdir(dir, { recursive: true });
|
||||
const ts = Date.now();
|
||||
const originalPath = join(dir, `${ts}-screenshot.png`);
|
||||
await this.daemon.saveScreenshot(originalPath);
|
||||
logger.info({ dir, files: [originalPath.split(/[\\/]/).pop()] }, 'Debug screenshot saved');
|
||||
return [originalPath];
|
||||
}
|
||||
|
||||
async saveRegion(region: Region, path: string): Promise<void> {
|
||||
await this.daemon.saveScreenshot(path, region);
|
||||
logger.info({ path, region }, 'Region screenshot saved');
|
||||
}
|
||||
|
||||
// ── Lifecycle ───────────────────────────────────────────────────────
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
await this.daemon.stop();
|
||||
}
|
||||
}
|
||||
90
src-old/game/WindowManager.ts
Normal file
90
src-old/game/WindowManager.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import koffi from 'koffi';
|
||||
import { logger } from '../util/logger.js';
|
||||
|
||||
// Win32 types
|
||||
const HWND = 'int';
|
||||
const BOOL = 'bool';
|
||||
const RECT = koffi.struct('RECT', {
|
||||
left: 'long',
|
||||
top: 'long',
|
||||
right: 'long',
|
||||
bottom: 'long',
|
||||
});
|
||||
|
||||
// Load user32.dll
|
||||
const user32 = koffi.load('user32.dll');
|
||||
|
||||
const FindWindowW = user32.func('FindWindowW', HWND, ['str16', 'str16']);
|
||||
const SetForegroundWindow = user32.func('SetForegroundWindow', BOOL, [HWND]);
|
||||
const ShowWindow = user32.func('ShowWindow', BOOL, [HWND, 'int']);
|
||||
const BringWindowToTop = user32.func('BringWindowToTop', BOOL, [HWND]);
|
||||
const GetForegroundWindow = user32.func('GetForegroundWindow', HWND, []);
|
||||
const GetWindowRect = user32.func('GetWindowRect', BOOL, [HWND, koffi.out(koffi.pointer(RECT))]);
|
||||
const IsWindow = user32.func('IsWindow', BOOL, [HWND]);
|
||||
const keybd_event = user32.func('keybd_event', 'void', ['uint8', 'uint8', 'uint32', 'uint']);
|
||||
const MapVirtualKeyW = user32.func('MapVirtualKeyW', 'uint32', ['uint32', 'uint32']);
|
||||
|
||||
// Constants
|
||||
const SW_RESTORE = 9;
|
||||
const VK_MENU = 0x12; // Alt key
|
||||
const KEYEVENTF_KEYUP = 0x0002;
|
||||
|
||||
export class WindowManager {
|
||||
private hwnd: number = 0;
|
||||
|
||||
constructor(private windowTitle: string) {}
|
||||
|
||||
findWindow(): number {
|
||||
this.hwnd = FindWindowW(null as unknown as string, this.windowTitle);
|
||||
if (this.hwnd === 0) {
|
||||
logger.warn({ title: this.windowTitle }, 'Window not found');
|
||||
} else {
|
||||
logger.info({ title: this.windowTitle, hwnd: this.hwnd }, 'Window found');
|
||||
}
|
||||
return this.hwnd;
|
||||
}
|
||||
|
||||
focusWindow(): boolean {
|
||||
if (!this.hwnd || !IsWindow(this.hwnd)) {
|
||||
this.findWindow();
|
||||
}
|
||||
if (!this.hwnd) return false;
|
||||
|
||||
// Restore if minimized
|
||||
ShowWindow(this.hwnd, SW_RESTORE);
|
||||
|
||||
// Alt-key trick to bypass SetForegroundWindow restriction
|
||||
const altScan = MapVirtualKeyW(VK_MENU, 0);
|
||||
keybd_event(VK_MENU, altScan, 0, 0);
|
||||
keybd_event(VK_MENU, altScan, KEYEVENTF_KEYUP, 0);
|
||||
|
||||
BringWindowToTop(this.hwnd);
|
||||
const result = SetForegroundWindow(this.hwnd);
|
||||
|
||||
if (!result) {
|
||||
logger.warn('SetForegroundWindow failed');
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
getWindowRect(): { left: number; top: number; right: number; bottom: number } | null {
|
||||
if (!this.hwnd || !IsWindow(this.hwnd)) {
|
||||
this.findWindow();
|
||||
}
|
||||
if (!this.hwnd) return null;
|
||||
|
||||
const rect = { left: 0, top: 0, right: 0, bottom: 0 };
|
||||
const success = GetWindowRect(this.hwnd, rect);
|
||||
if (!success) return null;
|
||||
return rect;
|
||||
}
|
||||
|
||||
isGameFocused(): boolean {
|
||||
const fg = GetForegroundWindow();
|
||||
return fg === this.hwnd && this.hwnd !== 0;
|
||||
}
|
||||
|
||||
getHwnd(): number {
|
||||
return this.hwnd;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue