NLP Cheatsheet

Preprocessing (NLTK / spaCy)

Tokenize, Clean, Lemmatize

from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, WordNetLemmatizer

word_tokenize(text)          # word tokens
sent_tokenize(text)          # sentence split
stop = set(stopwords.words("english"))
PorterStemmer().stem("running")        # 'run'
WordNetLemmatizer().lemmatize("mice")  # 'mouse'

# spaCy (fast, production)
import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("Ada worked in London.")
[(t.text, t.lemma_, t.pos_) for t in doc]
[(e.text, e.label_) for e in doc.ents]  # NER

Vectorizers (scikit-learn)

BoW & TF-IDF

from sklearn.feature_extraction.text import (
    CountVectorizer, TfidfVectorizer)

CountVectorizer(ngram_range=(1,2), stop_words="english")
TfidfVectorizer(max_features=5000, sublinear_tf=True)

X = vec.fit_transform(corpus)   # sparse matrix
vec.get_feature_names_out()
vec.transform(new_docs)

TF-IDF Formula

tf(t,d)   = count(t,d) / len(d)
idf(t)    = log(N / df(t)) + 1
tfidf     = tf(t,d) * idf(t)
cosine_sim(a,b) = (a . b) / (||a|| * ||b||)

Word Embeddings (gensim)

from gensim.models import Word2Vec, KeyedVectors

Word2Vec(sentences, vector_size=100, window=5,
         min_count=2, sg=1)      # sg=1 skip-gram, 0 CBOW
model.wv["word"]                 # vector
model.wv.most_similar("king", topn=5)
model.wv.similarity("cat", "dog")
# analogy: king - man + woman
model.wv.most_similar(positive=["king","woman"],
                      negative=["man"])

# Load pretrained GloVe / fastText
KeyedVectors.load_word2vec_format("glove.txt")

Hugging Face Pipelines

from transformers import pipeline

pipeline("sentiment-analysis")(text)
pipeline("ner", aggregation_strategy="simple")(text)
pipeline("question-answering")(question=q, context=c)
pipeline("summarization")(text, max_length=120)
pipeline("text-generation", model="gpt2")(prompt)
pipeline("zero-shot-classification")(
    text, candidate_labels=["sports","tech","food"])
pipeline("fill-mask")("Paris is the [MASK] of France.")
pipeline("translation_en_to_fr")(text)

Tokenizer + Model API

from transformers import (AutoTokenizer,
    AutoModelForSequenceClassification, AutoModelForCausalLM)
import torch

tok = AutoTokenizer.from_pretrained(name)
enc = tok(text, return_tensors="pt",
          padding=True, truncation=True, max_length=512)
tok.decode(enc["input_ids"][0])
tok.convert_ids_to_tokens(enc["input_ids"][0])

model = AutoModelForSequenceClassification.from_pretrained(name)
logits = model(**enc).logits
probs = torch.softmax(logits, dim=-1)

# Generation
gen = AutoModelForCausalLM.from_pretrained("gpt2")
out = gen.generate(**enc, max_new_tokens=50,
                   do_sample=True, top_p=0.9, temperature=0.7)

Fine-Tuning & LoRA

from transformers import TrainingArguments, Trainer
from peft import LoraConfig, get_peft_model

lora = LoraConfig(r=8, lora_alpha=16, lora_dropout=0.05,
                  target_modules=["q_proj","v_proj"])
model = get_peft_model(base_model, lora)

args = TrainingArguments(output_dir="out",
    learning_rate=2e-5, num_train_epochs=3,
    per_device_train_batch_size=16, eval_strategy="epoch")
Trainer(model, args, train_dataset=tr, eval_dataset=va).train()
model.push_to_hub("user/my-model")

Model Families

Attention

Attention(Q,K,V) = softmax(Q Kᵀ / sqrt(d_k)) V
MultiHead = concat(head_1..head_h) W_O
# add positional encoding to token embeddings