cruciverb/scripts/extract-grids.js

#!/usr/bin/env node
// Extract unique grid patterns from NYT crosswords JSON and xd puzzle files
// Output: one file per size (grids_15x15.txt, grids_21x21.txt)
// Format: each grid is lines of '#' (black) and '.' (white), separated by blank lines

const fs = require('fs')
const path = require('path')

const grids = new Map() // size -> Set of pattern strings

function addGrid(width, height, pattern) {
  // only keep 15x15 and 21x21
  if (!((width === 15 && height === 15) || (width === 21 && height === 21))) return
  const key = `${width}x${height}`
  if (!grids.has(key)) grids.set(key, new Set())
  grids.get(key).add(pattern)
}

function gridToPattern(grid, width, height) {
  const lines = []
  for (let r = 0; r < height; r++) {
    let line = ''
    for (let c = 0; c < width; c++) {
      const cell = grid[r * width + c]
      line += (cell === '.' || cell === '#') ? '#' : '.'
    }
    lines.push(line)
  }
  return lines.join('\n')
}

// validate pattern: every white cell in runs >= 3 both directions, connected
function isValidPattern(pattern, width, height) {
  const cells = pattern.split('\n').map(l => l.split(''))

  // check runs >= 3 in both directions for every white cell
  for (let r = 0; r < height; r++) {
    for (let c = 0; c < width; c++) {
      if (cells[r][c] !== '.') continue
      // across run length
      let as_ = c, ae = c
      while (as_ > 0 && cells[r][as_-1] === '.') as_--
      while (ae < width-1 && cells[r][ae+1] === '.') ae++
      if (ae - as_ + 1 < 3) return false
      // down run length
      let ds = r, de = r
      while (ds > 0 && cells[ds-1][c] === '.') ds--
      while (de < height-1 && cells[de+1][c] === '.') de++
      if (de - ds + 1 < 3) return false
    }
  }

  // connectivity check (BFS)
  let startR = -1, startC = -1, whiteCount = 0
  for (let r = 0; r < height && startR < 0; r++)
    for (let c = 0; c < width && startR < 0; c++)
      if (cells[r][c] === '.') { startR = r; startC = c }
  if (startR < 0) return false

  for (let r = 0; r < height; r++)
    for (let c = 0; c < width; c++)
      if (cells[r][c] === '.') whiteCount++

  const visited = Array.from({length: height}, () => Array(width).fill(false))
  const queue = [[startR, startC]]
  visited[startR][startC] = true
  let reached = 1
  while (queue.length > 0) {
    const [r, c] = queue.shift()
    for (const [dr, dc] of [[-1,0],[1,0],[0,-1],[0,1]]) {
      const nr = r+dr, nc = c+dc
      if (nr >= 0 && nr < height && nc >= 0 && nc < width && !visited[nr][nc] && cells[nr][nc] === '.') {
        visited[nr][nc] = true
        reached++
        queue.push([nr, nc])
      }
    }
  }
  return reached === whiteCount
}

// --- Process NYT JSON files ---
function processNYT(dir) {
  if (!fs.existsSync(dir)) { console.log('NYT dir not found:', dir); return }
  let count = 0
  const walk = (d) => {
    for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
      const full = path.join(d, entry.name)
      if (entry.isDirectory()) { walk(full); continue }
      if (!entry.name.endsWith('.json')) continue
      try {
        const data = JSON.parse(fs.readFileSync(full, 'utf8'))
        const w = data.size?.cols || 0
        const h = data.size?.rows || 0
        if ((w === 15 && h === 15) || (w === 21 && h === 21)) {
          if (data.grid && data.grid.length === w * h) {
            const pattern = gridToPattern(data.grid, w, h)
            if (isValidPattern(pattern, w, h)) {
              addGrid(w, h, pattern)
              count++
            }
          }
        }
      } catch {}
    }
  }
  walk(dir)
  console.log(`NYT: extracted ${count} valid grids`)
}

// --- Process xd files ---
function processXD(dir) {
  if (!fs.existsSync(dir)) { console.log('XD dir not found:', dir); return }
  let count = 0
  const walk = (d) => {
    for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
      const full = path.join(d, entry.name)
      if (entry.isDirectory()) { walk(full); continue }
      if (!entry.name.endsWith('.xd')) continue
      try {
        const text = fs.readFileSync(full, 'utf8')
        // xd format: headers, blank line, grid section, blank line, clues
        // grid lines are like: .ABC.DEF. where . = black, letters = white
        const sections = text.split(/\n\n+/)
        let gridSection = null
        for (const sec of sections) {
          const lines = sec.trim().split('\n').filter(l => l.trim())
          // grid section: all lines same length, contain only letters, dots, #, and spaces
          if (lines.length >= 5 && lines.every(l => /^[A-Za-z.#_]+$/.test(l.trim())) &&
              new Set(lines.map(l => l.trim().length)).size === 1) {
            gridSection = lines.map(l => l.trim())
            break
          }
        }
        if (!gridSection) continue
        const h = gridSection.length
        const w = gridSection[0].length
        if (!((w === 15 && h === 15) || (w === 21 && h === 21))) continue

        const patternLines = gridSection.map(line =>
          line.split('').map(c => (c === '.' || c === '#' || c === '_') ? '#' : '.').join('')
        )
        const pattern = patternLines.join('\n')
        if (isValidPattern(pattern, w, h)) {
          addGrid(w, h, pattern)
          count++
        }
      } catch {}
    }
  }
  walk(dir)
  console.log(`XD: extracted ${count} valid grids`)
}

// --- Main ---
const nytDir = process.argv[2] || '/tmp/nyt_crosswords'
const xdDir = process.argv[3] || '/tmp/xd-puzzles'
const outDir = process.argv[4] || 'data/grid-templates'

console.log('Processing NYT crosswords...')
processNYT(nytDir)

console.log('Processing xd puzzles...')
processXD(xdDir)

// Write output
fs.mkdirSync(outDir, { recursive: true })

for (const [size, patterns] of grids) {
  const outFile = path.join(outDir, `grids_${size}.txt`)
  const content = [...patterns].join('\n\n') + '\n'
  fs.writeFileSync(outFile, content)
  console.log(`${size}: ${patterns.size} unique patterns -> ${outFile}`)
}

console.log('Done!')