Skip to content

preprocess_texts

Preprocess texts: optionally split to sentences, filter.

If config is None or split_to_sentences is False, returns texts as-is (with optional length/char filtering).

When split_to_sentences is True or an integer:

  • If spacy_model is given: use spacy sentencizer.
  • Otherwise: use simple regex split on .!?
  • Integer values greater than 1 yield non-overlapping sentence windows.

Parameters:

Name Type Description Default
texts List[str]

Input text strings (paragraphs or documents).

 required
config Optional[TextPreprocessConfig]

TextPreprocessConfig. If None, returns texts as-is.

 None
spacy_model Optional[str]

Spacy model name for sentencizer (e.g. "de_core_news_sm"). Only used when split_to_sentences is truthy.

 None
download_if_missing bool

If True, download the spacy model if it is not found.

 True

Returns:

Type Description
List[str]

List of (optionally filtered) text strings.
Source code in gradiend/data/text/preprocess.py 
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
def preprocess_texts(
    texts: List[str],
    config: Optional[TextPreprocessConfig] = None,
    spacy_model: Optional[str] = None,
    download_if_missing: bool = True,
) -> List[str]:
    """Preprocess texts: optionally split to sentences, filter.

    If config is None or split_to_sentences is False, returns texts as-is
    (with optional length/char filtering).

    When split_to_sentences is True or an integer:

    - If spacy_model is given: use spacy sentencizer.
    - Otherwise: use simple regex split on .!?
    - Integer values greater than 1 yield non-overlapping sentence windows.

    Args:
        texts: Input text strings (paragraphs or documents).
        config: TextPreprocessConfig. If None, returns texts as-is.
        spacy_model: Spacy model name for sentencizer (e.g. "de_core_news_sm").
            Only used when split_to_sentences is truthy.
        download_if_missing: If True, download the spacy model if it is not found.

    Returns:
        List of (optionally filtered) text strings.
    """
    if not isinstance(texts, list):
        raise TypeError(f"texts must be a list, got {type(texts).__name__}")
    if not all(isinstance(x, str) for x in texts):
        raise TypeError("texts must contain only strings")
    if config is not None and not isinstance(config, TextPreprocessConfig):
        raise TypeError(f"config must be TextPreprocessConfig or None, got {type(config).__name__}")
    if spacy_model is not None and not isinstance(spacy_model, str):
        raise TypeError(f"spacy_model must be str or None, got {type(spacy_model).__name__}")
    if not isinstance(download_if_missing, bool):
        raise TypeError(f"download_if_missing must be bool, got {type(download_if_missing).__name__}")

    if config is None:
        return texts

    out: List[str] = []
    if config.split_to_sentences:
        sentences = _split_to_sentences(texts, spacy_model, download_if_missing, config.split_to_sentences)
    else:
        sentences = texts

    for s in sentences:
        s = s.strip()
        if not s:
            continue
        if config.min_chars is not None and len(s) < config.min_chars:
            continue
        if config.max_chars is not None and len(s) > config.max_chars:
            continue
        if config.exclude_chars and any(c in s for c in config.exclude_chars):
            continue
        if config.custom_filter is not None and not config.custom_filter(s):
            continue
        out.append(s)

    logger.debug(f"preprocess_texts: {len(texts)} -> {len(out)} items")
    return out