97 lines
2.6 KiB
Rust
97 lines
2.6 KiB
Rust
use std::path::Path;
|
|
|
|
use crate::error::Result;
|
|
|
|
use super::blocklist::Blocklist;
|
|
use super::word::Word;
|
|
|
|
pub fn load_wordlist(path: &Path, blocklist: &Blocklist, min_length: usize) -> Result<Vec<Word>> {
|
|
let content = std::fs::read_to_string(path)?;
|
|
let mut words = Vec::new();
|
|
|
|
for line in content.lines() {
|
|
let w = line.trim().to_uppercase();
|
|
if w.len() < min_length {
|
|
continue;
|
|
}
|
|
if !w.chars().all(|c| c.is_alphabetic()) {
|
|
continue;
|
|
}
|
|
if blocklist.is_blocked(&w) {
|
|
continue;
|
|
}
|
|
// frequency heuristic: shorter common words score higher
|
|
let freq = base_frequency(&w);
|
|
words.push(Word::new(w, freq));
|
|
}
|
|
|
|
// sort by length then alphabetically for deterministic ordering
|
|
words.sort_by(|a, b| a.length.cmp(&b.length).then(a.text.cmp(&b.text)));
|
|
words.dedup_by(|a, b| a.text == b.text);
|
|
|
|
Ok(words)
|
|
}
|
|
|
|
fn base_frequency(word: &str) -> f64 {
|
|
// simple heuristic: common lengths (3-8) get a boost, very long words score lower
|
|
// this is a placeholder until we have real frequency data
|
|
let len = word.len();
|
|
match len {
|
|
3 => 0.7,
|
|
4 => 0.75,
|
|
5 => 0.8,
|
|
6 => 0.75,
|
|
7 => 0.65,
|
|
8 => 0.55,
|
|
9 => 0.45,
|
|
10 => 0.35,
|
|
_ if len > 10 => 0.2,
|
|
_ => 0.5,
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn load_enable() {
|
|
let path = Path::new("../data/wordlists/enable.txt");
|
|
if !path.exists() {
|
|
return; // skip if wordlist not present
|
|
}
|
|
let bl = Blocklist::empty();
|
|
let words = load_wordlist(path, &bl, 3).unwrap();
|
|
assert!(words.len() > 100_000, "expected 100k+ words, got {}", words.len());
|
|
// all words should be uppercase, >= 3 chars, alpha only
|
|
for w in &words {
|
|
assert!(w.length >= 3);
|
|
assert!(w.text.chars().all(|c| c.is_alphabetic()), "bad word: {}", w.text);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn filters_short_words() {
|
|
let path = Path::new("../data/wordlists/enable.txt");
|
|
if !path.exists() {
|
|
return;
|
|
}
|
|
let bl = Blocklist::empty();
|
|
let words = load_wordlist(path, &bl, 5).unwrap();
|
|
for w in &words {
|
|
assert!(w.length >= 5);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn blocklist_filters() {
|
|
let path = Path::new("../data/wordlists/enable.txt");
|
|
if !path.exists() {
|
|
return;
|
|
}
|
|
let bl = Blocklist::empty();
|
|
let words = load_wordlist(path, &bl, 3).unwrap();
|
|
assert!(words.iter().any(|w| w.text == "THE"));
|
|
}
|
|
}
|