word loading from data files

cruciverb-core/src/dict/loader.rs

@@ -0,0 +1,96 @@
+use std::path::Path;
+
+use crate::error::Result;
+
+use super::blocklist::Blocklist;
+use super::word::Word;
+
+pub fn load_wordlist(path: &Path, blocklist: &Blocklist, min_length: usize) -> Result<Vec<Word>> {
+    let content = std::fs::read_to_string(path)?;
+    let mut words = Vec::new();
+
+    for line in content.lines() {
+        let w = line.trim().to_uppercase();
+        if w.len() < min_length {
+            continue;
+        }
+        if !w.chars().all(|c| c.is_alphabetic()) {
+            continue;
+        }
+        if blocklist.is_blocked(&w) {
+            continue;
+        }
+        // frequency heuristic: shorter common words score higher
+        let freq = base_frequency(&w);
+        words.push(Word::new(w, freq));
+    }
+
+    // sort by length then alphabetically for deterministic ordering
+    words.sort_by(|a, b| a.length.cmp(&b.length).then(a.text.cmp(&b.text)));
+    words.dedup_by(|a, b| a.text == b.text);
+
+    Ok(words)
+}
+
+fn base_frequency(word: &str) -> f64 {
+    // simple heuristic: common lengths (3-8) get a boost, very long words score lower
+    // this is a placeholder until we have real frequency data
+    let len = word.len();
+    match len {
+        3 => 0.7,
+        4 => 0.75,
+        5 => 0.8,
+        6 => 0.75,
+        7 => 0.65,
+        8 => 0.55,
+        9 => 0.45,
+        10 => 0.35,
+        _ if len > 10 => 0.2,
+        _ => 0.5,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn load_enable() {
+        let path = Path::new("../data/wordlists/enable.txt");
+        if !path.exists() {
+            return; // skip if wordlist not present
+        }
+        let bl = Blocklist::empty();
+        let words = load_wordlist(path, &bl, 3).unwrap();
+        assert!(words.len() > 100_000, "expected 100k+ words, got {}", words.len());
+        // all words should be uppercase, >= 3 chars, alpha only
+        for w in &words {
+            assert!(w.length >= 3);
+            assert!(w.text.chars().all(|c| c.is_alphabetic()), "bad word: {}", w.text);
+        }
+    }
+
+    #[test]
+    fn filters_short_words() {
+        let path = Path::new("../data/wordlists/enable.txt");
+        if !path.exists() {
+            return;
+        }
+        let bl = Blocklist::empty();
+        let words = load_wordlist(path, &bl, 5).unwrap();
+        for w in &words {
+            assert!(w.length >= 5);
+        }
+    }
+
+    #[test]
+    fn blocklist_filters() {
+        let path = Path::new("../data/wordlists/enable.txt");
+        if !path.exists() {
+            return;
+        }
+        let bl = Blocklist::empty();
+        let words = load_wordlist(path, &bl, 3).unwrap();
+        assert!(words.iter().any(|w| w.text == "THE"));
+    }
+}