namespace OcrDaemon; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; using System.Text.Json; using Tesseract; using SdImageFormat = System.Drawing.Imaging.ImageFormat; class OcrHandler(TesseractEngine engine) { private Bitmap? _referenceFrame; public object HandleOcr(Request req) { using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region); using var pix = ImageUtils.BitmapToPix(bitmap); using var page = engine.Process(pix); var text = page.GetText(); var lines = ImageUtils.ExtractLinesFromPage(page, offsetX: 0, offsetY: 0); return new OcrResponse { Text = text, Lines = lines }; } public object HandleScreenshot(Request req) { if (string.IsNullOrEmpty(req.Path)) return new ErrorResponse("screenshot command requires 'path'"); // If a reference frame exists, save that (same image used for diff-ocr). // Otherwise capture a new frame. var bitmap = _referenceFrame ?? ScreenCapture.CaptureOrLoad(req.File, req.Region); var format = ImageUtils.GetImageFormat(req.Path); var dir = Path.GetDirectoryName(req.Path); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) Directory.CreateDirectory(dir); bitmap.Save(req.Path, format); if (bitmap != _referenceFrame) bitmap.Dispose(); return new OkResponse(); } public object HandleCapture(Request req) { using var bitmap = ScreenCapture.CaptureOrLoad(req.File, req.Region); using var ms = new MemoryStream(); bitmap.Save(ms, SdImageFormat.Png); var base64 = Convert.ToBase64String(ms.ToArray()); return new CaptureResponse { Image = base64 }; } public object HandleSnapshot(Request req) { _referenceFrame?.Dispose(); _referenceFrame = ScreenCapture.CaptureOrLoad(req.File, req.Region); return new OkResponse(); } public object HandleDiffOcr(Request req) => HandleDiffOcr(req, new DiffOcrParams { DiffThresh = req.Threshold > 0 ? req.Threshold : 30, }); public object HandleDiffOcr(Request req, DiffOcrParams p) { if (_referenceFrame == null) return new ErrorResponse("No reference snapshot stored. Send 'snapshot' first."); using var current = ScreenCapture.CaptureOrLoad(req.File, null); int w = Math.Min(_referenceFrame.Width, current.Width); int h = Math.Min(_referenceFrame.Height, current.Height); // Get raw pixels for both frames var refData = _referenceFrame.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); byte[] refPx = new byte[refData.Stride * h]; Marshal.Copy(refData.Scan0, refPx, 0, refPx.Length); _referenceFrame.UnlockBits(refData); int stride = refData.Stride; var curData = current.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly, PixelFormat.Format32bppArgb); byte[] curPx = new byte[curData.Stride * h]; Marshal.Copy(curData.Scan0, curPx, 0, curPx.Length); current.UnlockBits(curData); // Detect pixels that got DARKER (tooltip = dark overlay). // This filters out item highlight glow (brighter) and cursor changes. int diffThresh = p.DiffThresh; bool[] changed = new bool[w * h]; int totalChanged = 0; for (int y = 0; y < h; y++) { for (int x = 0; x < w; x++) { int i = y * stride + x * 4; int darkerB = refPx[i] - curPx[i]; int darkerG = refPx[i + 1] - curPx[i + 1]; int darkerR = refPx[i + 2] - curPx[i + 2]; if (darkerB + darkerG + darkerR > diffThresh) { changed[y * w + x] = true; totalChanged++; } } } bool debug = req.Debug; if (totalChanged == 0) { if (debug) Console.Error.WriteLine(" diff-ocr: no changes detected"); return new OcrResponse { Text = "", Lines = [] }; } // Two-pass density detection: // Pass 1: Find row range using full-width row counts // Pass 2: Find column range using only pixels within detected row range // This makes the column threshold relative to tooltip height, not screen height. int maxGap = p.MaxGap; // Pass 1: count changed pixels per row, find longest active run int[] rowCounts = new int[h]; for (int y = 0; y < h; y++) for (int x = 0; x < w; x++) if (changed[y * w + x]) rowCounts[y]++; int rowThresh = w / p.RowThreshDiv; int bestRowStart = 0, bestRowEnd = 0, bestRowLen = 0; int curRowStart = -1, lastActiveRow = -1; for (int y = 0; y < h; y++) { if (rowCounts[y] >= rowThresh) { if (curRowStart < 0) curRowStart = y; lastActiveRow = y; } else if (curRowStart >= 0 && y - lastActiveRow > maxGap) { int len = lastActiveRow - curRowStart + 1; if (len > bestRowLen) { bestRowStart = curRowStart; bestRowEnd = lastActiveRow; bestRowLen = len; } curRowStart = -1; } } if (curRowStart >= 0) { int len = lastActiveRow - curRowStart + 1; if (len > bestRowLen) { bestRowStart = curRowStart; bestRowEnd = lastActiveRow; bestRowLen = len; } } // Pass 2: count changed pixels per column, but only within the detected row range int[] colCounts = new int[w]; for (int y = bestRowStart; y <= bestRowEnd; y++) for (int x = 0; x < w; x++) if (changed[y * w + x]) colCounts[x]++; int tooltipHeight = bestRowEnd - bestRowStart + 1; int colThresh = tooltipHeight / p.ColThreshDiv; int bestColStart = 0, bestColEnd = 0, bestColLen = 0; int curColStart = -1, lastActiveCol = -1; for (int x = 0; x < w; x++) { if (colCounts[x] >= colThresh) { if (curColStart < 0) curColStart = x; lastActiveCol = x; } else if (curColStart >= 0 && x - lastActiveCol > maxGap) { int len = lastActiveCol - curColStart + 1; if (len > bestColLen) { bestColStart = curColStart; bestColEnd = lastActiveCol; bestColLen = len; } curColStart = -1; } } if (curColStart >= 0) { int len = lastActiveCol - curColStart + 1; if (len > bestColLen) { bestColStart = curColStart; bestColEnd = lastActiveCol; bestColLen = len; } } // Log density detection results Console.Error.WriteLine($" diff-ocr: changed={totalChanged} rows={bestRowStart}-{bestRowEnd}({bestRowLen}) cols={bestColStart}-{bestColEnd}({bestColLen}) rowThresh={rowThresh} colThresh={colThresh}"); if (bestRowLen < 50 || bestColLen < 50) { Console.Error.WriteLine($" diff-ocr: no tooltip-sized region found (rows={bestRowLen}, cols={bestColLen})"); return new OcrResponse { Text = "", Lines = [] }; } int pad = 0; int minX = Math.Max(bestColStart - pad, 0); int minY = Math.Max(bestRowStart - pad, 0); int maxX = Math.Min(bestColEnd + pad, w - 1); int maxY = Math.Min(bestRowEnd + pad, h - 1); // Dynamic right-edge trim: if the rightmost columns are much sparser than // the tooltip body, trim them. This handles the ~5% of cases where ambient // noise extends the detected region slightly on the right. int colSpan = maxX - minX + 1; if (colSpan > 100) { // Compute median column density in the middle 50% of the range int q1 = minX + colSpan / 4; int q3 = minX + colSpan * 3 / 4; long midSum = 0; int midCount = 0; for (int x = q1; x <= q3; x++) { midSum += colCounts[x]; midCount++; } double avgMidDensity = (double)midSum / midCount; double cutoff = avgMidDensity * p.TrimCutoff; // Trim from right while below cutoff while (maxX > minX + 100 && colCounts[maxX] < cutoff) maxX--; } int rw = maxX - minX + 1; int rh = maxY - minY + 1; if (debug) Console.Error.WriteLine($" diff-ocr: tooltip region ({minX},{minY}) {rw}x{rh}"); // Simple crop of the tooltip region from the current frame (no per-pixel masking). // The top-hat preprocessing will handle suppressing background text. using var cropped = current.Clone(new Rectangle(minX, minY, rw, rh), PixelFormat.Format32bppArgb); // Save before/after preprocessing images if path is provided if (!string.IsNullOrEmpty(req.Path)) { var dir = Path.GetDirectoryName(req.Path); if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir)) Directory.CreateDirectory(dir); cropped.Save(req.Path, ImageUtils.GetImageFormat(req.Path)); if (debug) Console.Error.WriteLine($" diff-ocr: saved raw to {req.Path}"); } // Pre-process for OCR: top-hat + binarize + upscale using var processed = ImagePreprocessor.PreprocessForOcr(cropped, p.KernelSize, p.Upscale); // Save fullscreen and preprocessed versions alongside raw if (!string.IsNullOrEmpty(req.Path)) { var ext = Path.GetExtension(req.Path); var fullPath = Path.ChangeExtension(req.Path, ".full" + ext); current.Save(fullPath, ImageUtils.GetImageFormat(fullPath)); if (debug) Console.Error.WriteLine($" diff-ocr: saved fullscreen to {fullPath}"); var prePath = Path.ChangeExtension(req.Path, ".pre" + ext); processed.Save(prePath, ImageUtils.GetImageFormat(prePath)); if (debug) Console.Error.WriteLine($" diff-ocr: saved preprocessed to {prePath}"); } using var pix = ImageUtils.BitmapToPix(processed); using var page = engine.Process(pix); var text = page.GetText(); var lines = ImageUtils.ExtractLinesFromPage(page, offsetX: minX, offsetY: minY); return new DiffOcrResponse { Text = text, Lines = lines, Region = new RegionRect { X = minX, Y = minY, Width = rw, Height = rh }, }; } public object HandleTest(Request req) => RunTestCases(new DiffOcrParams(), verbose: true); public object HandleTune(Request req) { // Coordinate descent: optimize one parameter at a time, repeat until stable. var best = new DiffOcrParams(); double bestScore = ScoreParams(best); Console.Error.WriteLine($"\n=== Tuning start === baseline score={bestScore:F3} {best}\n"); // Define search ranges for each parameter var sweeps = new (string Name, int[] Values, Action Set)[] { ("diffThresh", [10, 15, 20, 25, 30, 40, 50, 60], (p, v) => p.DiffThresh = v), ("rowThreshDiv", [10, 15, 20, 25, 30, 40, 50, 60], (p, v) => p.RowThreshDiv = v), ("colThreshDiv", [5, 8, 10, 12, 15, 20, 25, 30], (p, v) => p.ColThreshDiv = v), ("maxGap", [5, 8, 10, 12, 15, 20, 25, 30], (p, v) => p.MaxGap = v), ("kernelSize", [11, 15, 19, 21, 25, 31, 35, 41], (p, v) => p.KernelSize = v), ("upscale", [1, 2, 3], (p, v) => p.Upscale = v), }; // trimCutoff needs double values — handle separately double[] trimValues = [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5]; int totalEvals = 0; const int maxRounds = 3; for (int round = 0; round < maxRounds; round++) { bool improved = false; Console.Error.WriteLine($"--- Round {round + 1} ---"); // Sweep integer params foreach (var (name, values, set) in sweeps) { Console.Error.Write($" {name}: "); int bestVal = 0; double bestValScore = -1; foreach (int v in values) { var trial = best.Clone(); set(trial, v); double score = ScoreParams(trial); totalEvals++; Console.Error.Write($"{v}={score:F3} "); if (score > bestValScore) { bestValScore = score; bestVal = v; } } Console.Error.WriteLine(); if (bestValScore > bestScore) { set(best, bestVal); bestScore = bestValScore; improved = true; Console.Error.WriteLine($" → {name}={bestVal} score={bestScore:F3}"); } } // Sweep trimCutoff { Console.Error.Write($" trimCutoff: "); double bestTrim = best.TrimCutoff; double bestTrimScore = bestScore; foreach (double v in trimValues) { var trial = best.Clone(); trial.TrimCutoff = v; double score = ScoreParams(trial); totalEvals++; Console.Error.Write($"{v:F2}={score:F3} "); if (score > bestTrimScore) { bestTrimScore = score; bestTrim = v; } } Console.Error.WriteLine(); if (bestTrimScore > bestScore) { best.TrimCutoff = bestTrim; bestScore = bestTrimScore; improved = true; Console.Error.WriteLine($" → trimCutoff={bestTrim:F2} score={bestScore:F3}"); } } Console.Error.WriteLine($" End of round {round + 1}: score={bestScore:F3} {best}"); if (!improved) break; } Console.Error.WriteLine($"\n=== Tuning done === evals={totalEvals} bestScore={bestScore:F3}\n {best}\n"); // Run verbose test with best params for final report var finalResult = RunTestCases(best, verbose: true); return new TuneResponse { BestScore = bestScore, BestParams = best, Iterations = totalEvals, }; } /// Score a param set: average match ratio across all test cases (0-1). private double ScoreParams(DiffOcrParams p) { var result = RunTestCases(p, verbose: false); if (result is TestResponse tr && tr.Total > 0) return tr.Results.Average(r => r.Score); return 0; } private object RunTestCases(DiffOcrParams p, bool verbose) { var tessdataDir = Path.Combine(AppContext.BaseDirectory, "tessdata"); var casesPath = Path.Combine(tessdataDir, "cases.json"); if (!File.Exists(casesPath)) return new ErrorResponse($"cases.json not found at {casesPath}"); var json = File.ReadAllText(casesPath); var cases = JsonSerializer.Deserialize>(json); if (cases == null || cases.Count == 0) return new ErrorResponse("No test cases found in cases.json"); var results = new List(); int passCount = 0; foreach (var tc in cases) { if (verbose) Console.Error.WriteLine($"\n=== Test: {tc.Id} ==="); var fullPath = Path.Combine(tessdataDir, tc.FullImage); var imagePath = Path.Combine(tessdataDir, tc.Image); if (!File.Exists(fullPath)) { if (verbose) Console.Error.WriteLine($" SKIP: full image not found: {fullPath}"); results.Add(new TestCaseResult { Id = tc.Id, Passed = false, Score = 0, Missed = tc.Expected }); continue; } if (!File.Exists(imagePath)) { if (verbose) Console.Error.WriteLine($" SKIP: tooltip image not found: {imagePath}"); results.Add(new TestCaseResult { Id = tc.Id, Passed = false, Score = 0, Missed = tc.Expected }); continue; } // Run the same pipeline: snapshot (reference) then diff-ocr (with tooltip) HandleSnapshot(new Request { File = fullPath }); var diffResult = HandleDiffOcr(new Request { File = imagePath, Debug = verbose }, p); // Extract actual lines from the response List actualLines; if (diffResult is DiffOcrResponse diffResp) actualLines = diffResp.Lines.Select(l => l.Text.Trim()).Where(l => l.Length > 0).ToList(); else if (diffResult is OcrResponse ocrResp) actualLines = ocrResp.Lines.Select(l => l.Text.Trim()).Where(l => l.Length > 0).ToList(); else { if (verbose) Console.Error.WriteLine($" ERROR: unexpected response type"); results.Add(new TestCaseResult { Id = tc.Id, Passed = false, Score = 0, Missed = tc.Expected }); continue; } // Fuzzy match expected vs actual var matched = new List(); var missed = new List(); var usedActual = new HashSet(); foreach (var expected in tc.Expected) { int bestIdx = -1; double bestSim = 0; for (int i = 0; i < actualLines.Count; i++) { if (usedActual.Contains(i)) continue; double sim = LevenshteinSimilarity(expected, actualLines[i]); if (sim > bestSim) { bestSim = sim; bestIdx = i; } } if (bestIdx >= 0 && bestSim >= 0.75) { matched.Add(expected); usedActual.Add(bestIdx); if (verbose && bestSim < 1.0) Console.Error.WriteLine($" ~ {expected} → {actualLines[bestIdx]} (sim={bestSim:F2})"); } else { missed.Add(expected); if (verbose) Console.Error.WriteLine($" MISS: {expected}" + (bestIdx >= 0 ? $" (best: {actualLines[bestIdx]}, sim={bestSim:F2})" : "")); } } var extra = actualLines.Where((_, i) => !usedActual.Contains(i)).ToList(); if (verbose) foreach (var e in extra) Console.Error.WriteLine($" EXTRA: {e}"); double score = tc.Expected.Count > 0 ? (double)matched.Count / tc.Expected.Count : 1.0; bool passed = missed.Count == 0; if (passed) passCount++; if (verbose) Console.Error.WriteLine($" Result: {(passed ? "PASS" : "FAIL")} matched={matched.Count}/{tc.Expected.Count} extra={extra.Count} score={score:F2}"); results.Add(new TestCaseResult { Id = tc.Id, Passed = passed, Score = score, Matched = matched, Missed = missed, Extra = extra, }); } if (verbose) Console.Error.WriteLine($"\n=== Summary: {passCount}/{cases.Count} passed ===\n"); return new TestResponse { Passed = passCount, Failed = cases.Count - passCount, Total = cases.Count, Results = results, }; } private static double LevenshteinSimilarity(string a, string b) { a = a.ToLowerInvariant(); b = b.ToLowerInvariant(); if (a == b) return 1.0; int la = a.Length, lb = b.Length; if (la == 0 || lb == 0) return 0.0; var d = new int[la + 1, lb + 1]; for (int i = 0; i <= la; i++) d[i, 0] = i; for (int j = 0; j <= lb; j++) d[0, j] = j; for (int i = 1; i <= la; i++) for (int j = 1; j <= lb; j++) { int cost = a[i - 1] == b[j - 1] ? 0 : 1; d[i, j] = Math.Min(Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), d[i - 1, j - 1] + cost); } return 1.0 - (double)d[la, lb] / Math.Max(la, lb); } }