src/splitter.rs


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

use std::vec::Vec;

pub fn split_to_words(data : String) -> Vec<String> {
    let mut v : Vec<String> = data
        .to_lowercase()
        .split_whitespace()
        .map(str::to_string).collect();

    for word in v.iter_mut() {
        word.retain(|c| !r#"{}[]#(),".;:?!'%|0123456789/\^"#.contains(c))
    }

    v.retain(|str| !str.is_empty());

    v
}