_tokenize_text

Internal helper

This page documents an internal implementation helper, not a primary public API.

Source code in src/fabricops_kit/governance.py

def _tokenize_text(text: str) -> set[str]:
    base = re.sub(r"([a-z0-9])([A-Z])", r"\1 \2", text or "")
    tokens = [t for t in re.split(r"[^a-zA-Z0-9]+", base.lower()) if t]
    return set(tokens)