use std::path::Path; use crate::error::Result; use super::blocklist::Blocklist; use super::word::Word; pub fn load_wordlist(path: &Path, blocklist: &Blocklist, min_length: usize) -> Result> { let content = std::fs::read_to_string(path)?; let mut words = Vec::new(); for line in content.lines() { let w = line.trim().to_uppercase(); if w.len() < min_length { continue; } if !w.chars().all(|c| c.is_alphabetic()) { continue; } if blocklist.is_blocked(&w) { continue; } // frequency heuristic: shorter common words score higher let freq = base_frequency(&w); words.push(Word::new(w, freq)); } // sort by length then alphabetically for deterministic ordering words.sort_by(|a, b| a.length.cmp(&b.length).then(a.text.cmp(&b.text))); words.dedup_by(|a, b| a.text == b.text); Ok(words) } fn base_frequency(word: &str) -> f64 { // simple heuristic: common lengths (3-8) get a boost, very long words score lower // this is a placeholder until we have real frequency data let len = word.len(); match len { 3 => 0.7, 4 => 0.75, 5 => 0.8, 6 => 0.75, 7 => 0.65, 8 => 0.55, 9 => 0.45, 10 => 0.35, _ if len > 10 => 0.2, _ => 0.5, } } #[cfg(test)] mod tests { use super::*; #[test] fn load_enable() { let path = Path::new("../data/wordlists/enable.txt"); if !path.exists() { return; // skip if wordlist not present } let bl = Blocklist::empty(); let words = load_wordlist(path, &bl, 3).unwrap(); assert!(words.len() > 100_000, "expected 100k+ words, got {}", words.len()); // all words should be uppercase, >= 3 chars, alpha only for w in &words { assert!(w.length >= 3); assert!(w.text.chars().all(|c| c.is_alphabetic()), "bad word: {}", w.text); } } #[test] fn filters_short_words() { let path = Path::new("../data/wordlists/enable.txt"); if !path.exists() { return; } let bl = Blocklist::empty(); let words = load_wordlist(path, &bl, 5).unwrap(); for w in &words { assert!(w.length >= 5); } } #[test] fn blocklist_filters() { let path = Path::new("../data/wordlists/enable.txt"); if !path.exists() { return; } let bl = Blocklist::empty(); let words = load_wordlist(path, &bl, 3).unwrap(); assert!(words.iter().any(|w| w.text == "THE")); } }