diff --git a/cruciverb-core/src/dict/loader.rs b/cruciverb-core/src/dict/loader.rs new file mode 100644 index 0000000..6857aef --- /dev/null +++ b/cruciverb-core/src/dict/loader.rs @@ -0,0 +1,96 @@ +use std::path::Path; + +use crate::error::Result; + +use super::blocklist::Blocklist; +use super::word::Word; + +pub fn load_wordlist(path: &Path, blocklist: &Blocklist, min_length: usize) -> Result> { + let content = std::fs::read_to_string(path)?; + let mut words = Vec::new(); + + for line in content.lines() { + let w = line.trim().to_uppercase(); + if w.len() < min_length { + continue; + } + if !w.chars().all(|c| c.is_alphabetic()) { + continue; + } + if blocklist.is_blocked(&w) { + continue; + } + // frequency heuristic: shorter common words score higher + let freq = base_frequency(&w); + words.push(Word::new(w, freq)); + } + + // sort by length then alphabetically for deterministic ordering + words.sort_by(|a, b| a.length.cmp(&b.length).then(a.text.cmp(&b.text))); + words.dedup_by(|a, b| a.text == b.text); + + Ok(words) +} + +fn base_frequency(word: &str) -> f64 { + // simple heuristic: common lengths (3-8) get a boost, very long words score lower + // this is a placeholder until we have real frequency data + let len = word.len(); + match len { + 3 => 0.7, + 4 => 0.75, + 5 => 0.8, + 6 => 0.75, + 7 => 0.65, + 8 => 0.55, + 9 => 0.45, + 10 => 0.35, + _ if len > 10 => 0.2, + _ => 0.5, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn load_enable() { + let path = Path::new("../data/wordlists/enable.txt"); + if !path.exists() { + return; // skip if wordlist not present + } + let bl = Blocklist::empty(); + let words = load_wordlist(path, &bl, 3).unwrap(); + assert!(words.len() > 100_000, "expected 100k+ words, got {}", words.len()); + // all words should be uppercase, >= 3 chars, alpha only + for w in &words { + assert!(w.length >= 3); + assert!(w.text.chars().all(|c| c.is_alphabetic()), "bad word: {}", w.text); + } + } + + #[test] + fn filters_short_words() { + let path = Path::new("../data/wordlists/enable.txt"); + if !path.exists() { + return; + } + let bl = Blocklist::empty(); + let words = load_wordlist(path, &bl, 5).unwrap(); + for w in &words { + assert!(w.length >= 5); + } + } + + #[test] + fn blocklist_filters() { + let path = Path::new("../data/wordlists/enable.txt"); + if !path.exists() { + return; + } + let bl = Blocklist::empty(); + let words = load_wordlist(path, &bl, 3).unwrap(); + assert!(words.iter().any(|w| w.text == "THE")); + } +}