recognition types and preprocessing

frontend/src/recognition/preprocess.ts

@@ -0,0 +1,137 @@
+import type { Stroke } from './types'
+
+interface BoundingBox {
+  minX: number
+  minY: number
+  maxX: number
+  maxY: number
+  width: number
+  height: number
+}
+
+export function getBoundingBox(strokes: Stroke[]): BoundingBox {
+  let minX = Infinity, minY = Infinity
+  let maxX = -Infinity, maxY = -Infinity
+
+  for (const stroke of strokes) {
+    for (const pt of stroke) {
+      if (pt.x < minX) minX = pt.x
+      if (pt.y < minY) minY = pt.y
+      if (pt.x > maxX) maxX = pt.x
+      if (pt.y > maxY) maxY = pt.y
+    }
+  }
+
+  return { minX, minY, maxX, maxY, width: maxX - minX, height: maxY - minY }
+}
+
+// normalize stroke coordinates to 0-1 range within bounding box
+export function normalizeStrokes(strokes: Stroke[]): Stroke[] {
+  const bb = getBoundingBox(strokes)
+  if (bb.width === 0 && bb.height === 0) return strokes
+
+  const scale = Math.max(bb.width, bb.height)
+  const offsetX = (scale - bb.width) / 2
+  const offsetY = (scale - bb.height) / 2
+
+  return strokes.map(stroke =>
+    stroke.map(pt => ({
+      ...pt,
+      x: (pt.x - bb.minX + offsetX) / scale,
+      y: (pt.y - bb.minY + offsetY) / scale,
+    }))
+  )
+}
+
+// render strokes to a grayscale image for ONNX (28x28)
+// torchvision handles EMNIST transpose internally, so model expects
+// normal orientation. apply same normalization as training: (x-0.5)/0.5
+export function strokesToImage28(strokes: Stroke[]): Float32Array {
+  const size = 28
+  const padding = 4
+  const inner = size - padding * 2
+  const data = new Float32Array(size * size)
+
+  const bb = getBoundingBox(strokes)
+  if (bb.width === 0 && bb.height === 0) return data
+
+  const scale = Math.max(bb.width, bb.height)
+  const offsetX = (scale - bb.width) / 2
+  const offsetY = (scale - bb.height) / 2
+
+  for (const stroke of strokes) {
+    for (let i = 1; i < stroke.length; i++) {
+      const x0 = ((stroke[i - 1].x - bb.minX + offsetX) / scale) * inner + padding
+      const y0 = ((stroke[i - 1].y - bb.minY + offsetY) / scale) * inner + padding
+      const x1 = ((stroke[i].x - bb.minX + offsetX) / scale) * inner + padding
+      const y1 = ((stroke[i].y - bb.minY + offsetY) / scale) * inner + padding
+
+      const dist = Math.sqrt((x1 - x0) ** 2 + (y1 - y0) ** 2)
+      const steps = Math.max(Math.ceil(dist * 2), 1)
+      for (let s = 0; s <= steps; s++) {
+        const t = s / steps
+        const px = Math.round(x0 + (x1 - x0) * t)
+        const py = Math.round(y0 + (y1 - y0) * t)
+        for (let dy = -1; dy <= 1; dy++) {
+          for (let dx = -1; dx <= 1; dx++) {
+            const ix = px + dx
+            const iy = py + dy
+            if (ix >= 0 && ix < size && iy >= 0 && iy < size) {
+              data[iy * size + ix] = 1.0
+            }
+          }
+        }
+      }
+    }
+  }
+
+  // transpose (swap rows and columns) to match model's expected orientation
+  const out = new Float32Array(size * size)
+  for (let r = 0; r < size; r++) {
+    for (let c = 0; c < size; c++) {
+      out[c * size + r] = (data[r * size + c] - 0.5) / 0.5
+    }
+  }
+
+  return out
+}
+
+// render strokes to a canvas ImageData for Tesseract (black on white, larger)
+export function strokesToCanvas(strokes: Stroke[], size: number = 128): ImageData {
+  const canvas = new OffscreenCanvas(size, size)
+  const ctx = canvas.getContext('2d')!
+
+  // white background
+  ctx.fillStyle = 'white'
+  ctx.fillRect(0, 0, size, size)
+
+  const bb = getBoundingBox(strokes)
+  if (bb.width === 0 && bb.height === 0) {
+    return ctx.getImageData(0, 0, size, size)
+  }
+
+  const padding = 10
+  const inner = size - padding * 2
+  const scale = Math.max(bb.width, bb.height)
+
+  ctx.strokeStyle = 'black'
+  ctx.lineWidth = 4
+  ctx.lineCap = 'round'
+  ctx.lineJoin = 'round'
+
+  for (const stroke of strokes) {
+    if (stroke.length < 2) continue
+    ctx.beginPath()
+    const x0 = ((stroke[0].x - bb.minX) / scale) * inner + padding
+    const y0 = ((stroke[0].y - bb.minY) / scale) * inner + padding
+    ctx.moveTo(x0, y0)
+    for (let i = 1; i < stroke.length; i++) {
+      const x = ((stroke[i].x - bb.minX) / scale) * inner + padding
+      const y = ((stroke[i].y - bb.minY) / scale) * inner + padding
+      ctx.lineTo(x, y)
+    }
+    ctx.stroke()
+  }
+
+  return ctx.getImageData(0, 0, size, size)
+}

frontend/src/recognition/types.ts

@@ -0,0 +1,30 @@
+export interface StrokePoint {
+  x: number
+  y: number
+  pressure: number
+  timestamp: number
+}
+
+export type Stroke = StrokePoint[]
+
+export interface RecognitionCandidate {
+  letter: string
+  confidence: number
+}
+
+export interface EngineResult {
+  engine: 'onnx' | 'tesseract' | 'stroke'
+  candidates: RecognitionCandidate[]
+}
+
+export interface EnsembleResult {
+  candidates: RecognitionCandidate[]
+  decision: 'auto-fill' | 'disambiguate' | 'redraw'
+}
+
+export interface RecognitionEngine {
+  name: string
+  init(): Promise<void>
+  recognize(strokes: Stroke[]): Promise<RecognitionCandidate[]>
+  isReady(): boolean
+}