diff --git a/frontend/src/recognition/preprocess.ts b/frontend/src/recognition/preprocess.ts new file mode 100644 index 0000000..10c3ca7 --- /dev/null +++ b/frontend/src/recognition/preprocess.ts @@ -0,0 +1,137 @@ +import type { Stroke } from './types' + +interface BoundingBox { + minX: number + minY: number + maxX: number + maxY: number + width: number + height: number +} + +export function getBoundingBox(strokes: Stroke[]): BoundingBox { + let minX = Infinity, minY = Infinity + let maxX = -Infinity, maxY = -Infinity + + for (const stroke of strokes) { + for (const pt of stroke) { + if (pt.x < minX) minX = pt.x + if (pt.y < minY) minY = pt.y + if (pt.x > maxX) maxX = pt.x + if (pt.y > maxY) maxY = pt.y + } + } + + return { minX, minY, maxX, maxY, width: maxX - minX, height: maxY - minY } +} + +// normalize stroke coordinates to 0-1 range within bounding box +export function normalizeStrokes(strokes: Stroke[]): Stroke[] { + const bb = getBoundingBox(strokes) + if (bb.width === 0 && bb.height === 0) return strokes + + const scale = Math.max(bb.width, bb.height) + const offsetX = (scale - bb.width) / 2 + const offsetY = (scale - bb.height) / 2 + + return strokes.map(stroke => + stroke.map(pt => ({ + ...pt, + x: (pt.x - bb.minX + offsetX) / scale, + y: (pt.y - bb.minY + offsetY) / scale, + })) + ) +} + +// render strokes to a grayscale image for ONNX (28x28) +// torchvision handles EMNIST transpose internally, so model expects +// normal orientation. apply same normalization as training: (x-0.5)/0.5 +export function strokesToImage28(strokes: Stroke[]): Float32Array { + const size = 28 + const padding = 4 + const inner = size - padding * 2 + const data = new Float32Array(size * size) + + const bb = getBoundingBox(strokes) + if (bb.width === 0 && bb.height === 0) return data + + const scale = Math.max(bb.width, bb.height) + const offsetX = (scale - bb.width) / 2 + const offsetY = (scale - bb.height) / 2 + + for (const stroke of strokes) { + for (let i = 1; i < stroke.length; i++) { + const x0 = ((stroke[i - 1].x - bb.minX + offsetX) / scale) * inner + padding + const y0 = ((stroke[i - 1].y - bb.minY + offsetY) / scale) * inner + padding + const x1 = ((stroke[i].x - bb.minX + offsetX) / scale) * inner + padding + const y1 = ((stroke[i].y - bb.minY + offsetY) / scale) * inner + padding + + const dist = Math.sqrt((x1 - x0) ** 2 + (y1 - y0) ** 2) + const steps = Math.max(Math.ceil(dist * 2), 1) + for (let s = 0; s <= steps; s++) { + const t = s / steps + const px = Math.round(x0 + (x1 - x0) * t) + const py = Math.round(y0 + (y1 - y0) * t) + for (let dy = -1; dy <= 1; dy++) { + for (let dx = -1; dx <= 1; dx++) { + const ix = px + dx + const iy = py + dy + if (ix >= 0 && ix < size && iy >= 0 && iy < size) { + data[iy * size + ix] = 1.0 + } + } + } + } + } + } + + // transpose (swap rows and columns) to match model's expected orientation + const out = new Float32Array(size * size) + for (let r = 0; r < size; r++) { + for (let c = 0; c < size; c++) { + out[c * size + r] = (data[r * size + c] - 0.5) / 0.5 + } + } + + return out +} + +// render strokes to a canvas ImageData for Tesseract (black on white, larger) +export function strokesToCanvas(strokes: Stroke[], size: number = 128): ImageData { + const canvas = new OffscreenCanvas(size, size) + const ctx = canvas.getContext('2d')! + + // white background + ctx.fillStyle = 'white' + ctx.fillRect(0, 0, size, size) + + const bb = getBoundingBox(strokes) + if (bb.width === 0 && bb.height === 0) { + return ctx.getImageData(0, 0, size, size) + } + + const padding = 10 + const inner = size - padding * 2 + const scale = Math.max(bb.width, bb.height) + + ctx.strokeStyle = 'black' + ctx.lineWidth = 4 + ctx.lineCap = 'round' + ctx.lineJoin = 'round' + + for (const stroke of strokes) { + if (stroke.length < 2) continue + ctx.beginPath() + const x0 = ((stroke[0].x - bb.minX) / scale) * inner + padding + const y0 = ((stroke[0].y - bb.minY) / scale) * inner + padding + ctx.moveTo(x0, y0) + for (let i = 1; i < stroke.length; i++) { + const x = ((stroke[i].x - bb.minX) / scale) * inner + padding + const y = ((stroke[i].y - bb.minY) / scale) * inner + padding + ctx.lineTo(x, y) + } + ctx.stroke() + } + + return ctx.getImageData(0, 0, size, size) +} diff --git a/frontend/src/recognition/types.ts b/frontend/src/recognition/types.ts new file mode 100644 index 0000000..570c078 --- /dev/null +++ b/frontend/src/recognition/types.ts @@ -0,0 +1,30 @@ +export interface StrokePoint { + x: number + y: number + pressure: number + timestamp: number +} + +export type Stroke = StrokePoint[] + +export interface RecognitionCandidate { + letter: string + confidence: number +} + +export interface EngineResult { + engine: 'onnx' | 'tesseract' | 'stroke' + candidates: RecognitionCandidate[] +} + +export interface EnsembleResult { + candidates: RecognitionCandidate[] + decision: 'auto-fill' | 'disambiguate' | 'redraw' +} + +export interface RecognitionEngine { + name: string + init(): Promise + recognize(strokes: Stroke[]): Promise + isReady(): boolean +}