Tokenization

import re

# Whitespace split
tokens = "Hello world. NLP is fun.".split()

# Regex word tokenizer
words = re.findall(r"\b\w+\b", "Hello, world! NLP rocks.")

# NLTK tokenizers
from nltk.tokenize import word_tokenize, sent_tokenize
import nltk
nltk.download("punkt_tab", quiet=True)
sents = sent_tokenize("First sentence. Second one!")
tokens = word_tokenize("Don't go there.")

# spaCy tokenization
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("Apple is looking at buying a startup.")
spacy_tokens = [t.text for t in doc]
lemmas = [t.lemma_ for t in doc]

# Subword tokenization with HuggingFace
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained("bert-base-uncased")
ids = tok.encode("Tokenization is fun.", add_special_tokens=True)
print(tok.convert_ids_to_tokens(ids))