NLP チートシート

# install NLTK and spaCy
pip install nltk spacy
python -m spacy download en_core_web_sm

import nltk
import spacy

# download NLTK data
nltk.download('punkt')
nltk.download('stopwords')

# load a spaCy model
nlp = spacy.load("en_core_web_sm")
doc = nlp("Apple is looking at buying a U.K. startup.")
for token in doc:
    print(token.text, token.pos_, token.dep_)

import spacy

# blank pipeline (no pretrained components)
nlp_blank = spacy.blank("en")

# full pipeline: tokenizer -> tagger -> parser -> ner -> ...
nlp = spacy.load("en_core_web_sm")
print(nlp.pipe_names)   # ['tok2vec', 'tagger', 'parser', 'ner', ...]

# disable components you don't need (faster)
nlp_fast = spacy.load("en_core_web_sm", disable=["parser", "ner"])

# process many texts efficiently
docs = list(nlp.pipe(["Hello world.", "SpaCy is fast."]))
# stream from file with nlp.pipe(texts, batch_size=1000)

import nltk

# one-time downloads (cached in ~/nltk_data)
nltk.download('gutenberg')     # Project Gutenberg books
nltk.download('reuters')       # Reuters news corpus
nltk.download('brown')         # Brown corpus (categorized)
nltk.download('wordnet')       # lexical database
nltk.download('averaged_perceptron_tagger')

from nltk.corpus import gutenberg, brown

# first sentences of Moby Dick
print(gutenberg.sents('melville-moby_dick.txt')[0][:8])

# Brown corpus categories
print(brown.categories())      # ['news', 'editorial', 'reviews', ...]
news_words = brown.words(categories='news')

import spacy

nlp = spacy.load("en_core_web_sm")
text = "The quick brown fox jumps over the lazy dog. The dog barks."
doc = nlp(text)

# basic stats
print("Tokens:", len(doc))
print("Sentences:", len(list(doc.sents)))
print("Unique words:", len({t.lemma_.lower() for t in doc if t.is_alpha}))

# word frequency
from collections import Counter
words = [t.text.lower() for t in doc if not t.is_stop and t.is_alpha]
print(Counter(words).most_common(5))

import re
from pathlib import Path

# read with proper encoding
raw = Path("doc.txt").read_text(encoding="utf-8")

# normalize whitespace
text = re.sub(r"\s+", " ", raw).strip()

# remove HTML tags
text = re.sub(r"<[^>]+>", " ", text)

# fix curly quotes / dashes to ASCII
text = text.replace("\u2019", "'").replace("\u201c", '"')
text = text.replace("\u2013", "-").replace("\u2014", "-")

# lowercase only if case-insensitive task
clean = text.lower()

from nltk.tokenize import word_tokenize, sent_tokenize

text = "Hello there! How are you? I'm fine, thanks."

# sentence split (handles abbreviations)
sents = sent_tokenize(text)
# ['Hello there!', 'How are you?', "I'm fine, thanks."]

# word split (handles contractions, punctuation)
tokens = word_tokenize(text)
# ['Hello', 'there', '!', 'How', 'are', 'you', '?',
#  'I', "'m", 'fine', ',', 'thanks', '.']

# treebank tokenizer: 'don't' -> ['do', "n't"]
from nltk.tokenize import TreebankWordTokenizer
print(TreebankWordTokenizer().tokenize("don't stop"))

import spacy

nlp = spacy.load("en_core_web_sm")
doc = nlp("Let's go to N.Y.C. on 03/14/2023 for $5.50!")

# tokens keep their position in the original text
for tok in doc:
    print(f"{tok.text!r:12} start={tok.idx} end={tok.idx + len(tok.text)}")

# spans: contiguous slices of the Doc
span = doc[2:5]   # 'go to N.Y.C.'
print(span.text)

# spaCy never loses alignment to the original string
print(doc.text == "".join(t.text_with_ws for t in doc))  # True

from nltk.tokenize import RegexpTokenizer

# words only (drop punctuation)
tok = RegexpTokenizer(r"\w+")
print(tok.tokenize("Hello, world! 123"))   # ['Hello', 'world', '123']

# keep contractions: \w+|'\w+
tok2 = RegexpTokenizer(r"\w+|'\w+")
print(tok2.tokenize("don't stop"))          # ['do', "n't", 'stop']

# whitespace tokenizer (keeps punctuation attached)
from nltk.tokenize import WhitespaceTokenizer
print(WhitespaceTokenizer().tokenize("Hi! Bye."))   # ['Hi!', 'Bye.']

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace

# train a Byte-Pair-Encoding tokenizer
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Whitespace()
trainer = BpeTrainer(
    special_tokens=["[UNK]", "[PAD]", "[CLS]", "[SEP]"],
    vocab_size=8000,
)
tokenizer.train(files=["corpus.txt"], trainer=trainer)

enc = tokenizer.encode("Tokenization is fun.")
print(enc.tokens)        # ['Tok', 'eni', 'zer', ...]
print(enc.ids)           # [34, 912, ...]

import spacy
from spacy.tokenizer import Tokenizer
from spacy.util import compile_infix_regex, compile_prefix_regex

nlp = spacy.load("en_core_web_sm")

# add a special-case rule
specials = {"gimme": [{ORTH: "gim"}, {ORTH: "me"}]}
for text, pattern in specials.items():
    nlp.tokenizer.add_special_case(text, pattern)

print([t.text for t in nlp("gimme that")])   # ['gim', 'me', 'that']

# customize infixes (e.g. split on hyphens)
infixes = compile_infix_regex([r"-"])
nlp.tokenizer.infix_finditer = infixes.finditer
print([t.text for t in nlp("state-of-the-art")])

from transformers import AutoTokenizer

# load the tokenizer matching a pretrained model
tok = AutoTokenizer.from_pretrained("bert-base-uncased")

enc = tok("Tokenization with BERT.", return_tensors="pt")
print(enc.keys())         # dict_keys(['input_ids', 'token_type_ids', 'attention_mask'])
print(enc["input_ids"])   # tensor([[101, 19204, 3989, 102]])
print(tok.convert_ids_to_tokens(enc["input_ids"][0]))
# ['[CLS]', 'tokenization', 'with', 'bert', '.', '[SEP]']

# handle long inputs with sliding window
print(tok("Very long text", truncation=True, max_length=512))

import nltk
from nltk.tokenize import word_tokenize

nltk.download('averaged_perceptron_tagger_eng')

tokens = word_tokenize("They refuse to permit us to obtain the permit.")
tags = nltk.pos_tag(tokens)
# [('They', 'PRP'), ('refuse', 'VBP'), ('to', 'TO'),
#  ('permit', 'VB'), ..., ('permit', 'NN'), ('.', '.')]

# note: the two 'permit' tokens get different tags (VB verb vs NN noun)
for word, tag in tags:
    print(f"{word:10} {tag}")

import spacy
nlp = spacy.load("en_core_web_sm")

doc = nlp("The children were playing happily.")
for tok in doc:
    print(f"{tok.text:12} POS={tok.pos_:6} TAG={tok.tag_:5} "
          f"lemma={tok.lemma_:10} morph={tok.morph}")

# tok.pos_  -> coarse Universal POS (NOUN, VERB, ADJ, ...)
# tok.tag_  -> fine-grained Treebank tag (NN, VBD, JJ, ...)
# tok.morph -> morphological features (Tense=Past, Number=Plur, ...)

# Universal Dependencies tagset (17 tags, same across languages)
# ADJ   - adjective       | ADP   - adposition (prep/post)
# ADV   - adverb          | AUX   - auxiliary (is, has)
# CCONJ - coordinating conj | DET  - determiner (the, a)
# INTJ  - interjection    | NOUN  - noun
# NUM   - numeral         | PART  - particle ('s, not)
# PRON  - pronoun         | PROPN - proper noun
# PUNCT - punctuation     | SCONJ - subordinating conj
# SYM   - symbol          | VERB  - verb
# X     - other

import spacy
nlp = spacy.load("en_core_web_sm")
doc = nlp("She quickly ran to the store.")
nouns = [t.text for t in doc if t.pos_ in ("NOUN", "PROPN")]
verbs = [t.text for t in doc if t.pos_ == "VERB"]

import nltk
from nltk.corpus import brown, treebank

# Brown: each word pre-tagged
sent = brown.tagged_sents(categories='news')[0]
print(sent)   # [('The', 'AT'), ('Fulton', 'NP-TL'), ...]

# Penn Treebank ( Wall Street Journal)
for s in treebank.tagged_sents()[:1]:
    print(s)

# count tag distribution
from collections import Counter
tags = [tag for w, tag in brown.tagged_words(categories='news')]
print(Counter(tags).most_common(5))

from nltk.tag import UnigramTagger, DefaultTagger
from nltk.corpus import brown

# fallback: tag everything as NN
default = DefaultTagger('NN')

# unigram: most-likely tag per word (from training data)
train = brown.tagged_sents(categories='news')[:3000]
uni = UnigramTagger(train, backoff=default)

# backoff chain: unigram -> default
test = brown.tagged_sents(categories='news')[3000:3100]
print(f"Accuracy: {uni.evaluate(test):.3f}")

# regex tagger for patterns
from nltk.tag import RegexpTagger
regex = RegexpTagger([(r'^-?[0-9]+(\.[0-9]+)?$', 'CD')], backoff=default)

import spacy
nlp = spacy.load("en_core_web_sm")

doc = nlp("Apple is buying a U.K. startup for $1 billion in New York.")

for ent in doc.ents:
    print(f"{ent.text:20} {ent.label_:10} {spacy.explain(ent.label_)}")
# Apple              ORG        Companies, agencies, institutions
# U.K.               GPE        Countries, cities, states
# $1 billion         MONEY      Monetary values
# New York           GPE        Countries, cities, states

# visualize
from spacy import displacy
displacy.serve(doc, style="ent")  # opens a browser

import nltk
from nltk.tokenize import word_tokenize, sent_tokenize

text = "Barack Obama was born in Hawaii and led the USA."
sents = sent_tokenize(text)

for s in sents:
    tokens = word_tokenize(s)
    tags = nltk.pos_tag(tokens)
    # ne_chunk returns a nested tree of named entities
    tree = nltk.ne_chunk(tags)
    print(tree)
    # (S
    #   (PERSON Barack/NNP Obama/NNP)
    #   was/VBD born/VBN in/IN
    #   (GPE Hawaii/NNP) ...)

# extract just the named entities
for node in tree:
    if hasattr(node, 'label'):
        name = " ".join(w for w, t in node)
        print(node.label(), name)

import spacy
nlp = spacy.load("en_core_web_sm")

# list all labels the model knows
for label in nlp.get_pipe("ner").labels:
    print(f"{label:12} {spacy.explain(label)}")
# PERSON       People, including fictional
# NORP         Nationalities or religious/political groups
# FAC          Buildings, airports, highways
# ORG          Companies, agencies, institutions
# GPE          Countries, cities, states
# LOC          Non-GPE locations: mountain ranges, bodies of water
# PRODUCT      Vehicles, food, clothing (not services)
# EVENT        Battles, wars, sports events
# DATE         Absolute or relative dates
# TIME         Times smaller than a day
# MONEY        Monetary values
# QUANTITY     Measurements (weight, distance)
# ORDINAL      First, second, ...
# CARDINAL     Numerals that are not dates or amounts

import spacy
from spacy.training import Example

# start from a blank model (or fine-tune a pretrained one)
nlp = spacy.blank("en")
ner = nlp.add_pipe("ner")
ner.add_label("DRUG")

# training data: text + character offsets of entities
TRAIN = [
    ("Take aspirin daily.", {"entities": [(5, 11, "DRUG")]}),
    ("Ibuprofen reduces fever.", {"entities": [(0, 9, "DRUG")]}),
]

optimizer = nlp.initialize()
for epoch in range(30):
    for text, ann in TRAIN:
        doc = nlp.make_doc(text)
        nlp.update([Example.from_dict(doc, ann)], sgd=optimizer)

# save and use
nlp.to_disk("custom_ner")
doc = nlp("Give metformin 500mg.")
print([(e.text, e.label_) for e in doc.ents])  # [('metformin', 'DRUG')]

# Entity linking maps a mention to a unique ID in a knowledge base
# (e.g. "Apple" -> Apple Inc. the company, not the fruit)

import spacy
nlp = spacy.load("en_core_web_sm")

# spaCy's EntityLinker requires a knowledge base (KB).
# Building one from Wikipedia/Wikidata is involved; in practice use:

# Option 1: spaCy + Wikipedia-based KB (spacy-entity-linker)
# pip install spacy-entity-linker
# from spacy_entity_linker import EntityLinker
# linker = EntityLinker()
# nlp.add_pipe("entityLinker", last=True)

# Option 2: a dedicated library
# import entitylinking  # or use a Wikidata API directly

# Example query via Wikidata SPARQL
import requests
url = "https://www.wikidata.org/w/api.php"
params = {"action": "wbsearchentities", "search": "Apple",
          "language": "en", "format": "json"}
print(requests.get(url, params=params).json())

from nltk.stem import PorterStemmer

stemmer = PorterStemmer()
for w in ["running", "runs", "ran", "easily", "fairness", "studies"]:
    print(f"{w:10} -> {stemmer.stem(w)}")
# running    -> run
# runs       -> run
# ran        -> ran        (Porter doesn't handle irregulars)
# easily     -> easili     (sometimes ugly)
# fairness   -> fair
# studies    -> studi

# fast, rule-based, deterministic
print([stemmer.stem(t) for t in "the cats are running".split()])

from nltk.stem import SnowballStemmer

# Snowball = improved Porter, supports many languages
print(SnowballStemmer.languages)
# ['arabic', 'danish', 'dutch', 'english', 'finnish', 'french',
#  'german', 'hungarian', 'italian', 'norwegian', 'porter',
#  'portuguese', 'romanian', 'russian', 'spanish', 'swedish']

en = SnowballStemmer("english")
print(en.stem("running"), en.stem("fairly"), en.stem("studies"))
# run fair studi

# compare two languages
fr = SnowballStemmer("french")
print(fr.stem("mangerons"))   # mang

import nltk
from nltk.stem import WordNetLemmatizer
nltk.download('wordnet')

lem = WordNetLemmatizer()

# without POS, defaults to NOUN
print(lem.lemmatize("running"))            # running (treated as noun)
print(lem.lemmatize("running", pos='v'))   # run      (verb)
print(lem.lemmatize("better", pos='a'))    # good     (adjective)
print(lem.lemmatize("mice"))               # mouse    (irregular plural)

# pipeline: tokenize -> POS tag -> lemmatize with correct POS
from nltk.tokenize import word_tokenize
from nltk import pos_tag
words = word_tokenize("The cats were running quickly.")
treebank_to_wordnet = {'N': 'n', 'V': 'v', 'J': 'a', 'R': 'r'}
for w, t in pos_tag(words):
    wn_pos = treebank_to_wordnet.get(t[0], 'n')
    print(w, '->', lem.lemmatize(w, pos=wn_pos))

import spacy
nlp = spacy.load("en_core_web_sm")

doc = nlp("The cats were running faster. I ate two apples.")
for tok in doc:
    if tok.text.lower() != tok.lemma_:
        print(f"{tok.text:10} -> {tok.lemma_}")
# cats       -> cat
# were       -> be
# running    -> run
# faster     -> fast
# ate        -> eat
# apples     -> apple

# context-aware, handles irregulars automatically
# use lemma_ instead of text for bag-of-words features
docs_tokens = [[t.lemma_.lower() for t in nlp(d) if not t.is_stop]
               for d in ["Cats run.", "The cat ran."]]

import spacy
from nltk.stem import PorterStemmer
nlp = spacy.load("en_core_web_sm")
stem = PorterStemmer()

words = ["studies", "studying", "better", "feet", "running", "happily"]

print(f"{'word':12}{'stem':12}{'lemma':12}")
for w in words:
    lemma = nlp(w)[0].lemma_
    print(f"{w:12}{stem.stem(w):12}{lemma:12}")
# studies     studi       study
# studying    studi       study
# better      better      good      <- lemma handles irregular
# feet        feet        foot
# running     run         run
# happily     happili     happily

# rule of thumb:
#   search/indexing/topic-models -> stemming (fast)
#   interpretation/visualization -> lemmatization (accurate)

import nltk
from nltk.corpus import stopwords
nltk.download('stopwords')

# 18 languages available
print(stopwords.fileids())
# ['arabic', 'english', 'french', 'german', 'spanish', ...]

en = set(stopwords.words('english'))
print(len(en))                # 179
print("the" in en)            # True
print("python" in en)         # False

# remove from a token list
from nltk.tokenize import word_tokenize
text = "The quick brown fox is very lazy indeed"
tokens = [w for w in word_tokenize(text.lower()) if w.isalpha()]
filtered = [w for w in tokens if w not in en]
print(filtered)   # ['quick', 'brown', 'fox', 'lazy']

import spacy
nlp = spacy.load("en_core_web_sm")

# access the stopword set
print(len(nlp.Defaults.stop_words))        # 326
print("the" in nlp.Defaults.stop_words)    # True

# tokens expose is_stop directly
doc = nlp("The quick fox isn't lazy")
for tok in doc:
    print(f"{tok.text:8} is_stop={tok.is_stop}")
# The      is_stop=True
# quick    is_stop=False
# isn't    is_stop=True   (contraction included)

# content words only
content = [t.text for t in doc if not t.is_stop and not t.is_punct]

import spacy
nlp = spacy.load("en_core_web_sm")

def remove_stopwords(text, keep_negations=True):
    doc = nlp(text)
    out = []
    for tok in doc:
        if tok.is_stop or tok.is_punct:
            # keep negations for sentiment tasks
            if keep_negations and tok.lemma_ in ("not", "no", "never"):
                out.append(tok.text)
                continue
            continue
        out.append(tok.text)
    return " ".join(out)

print(remove_stopwords("I do not like green eggs and ham."))
# 'not like green eggs ham .'

# bigrams preserve phrases that stopwords broke:
# 'New York' should NOT lose its inner stopwords...

import spacy
nlp = spacy.load("en_core_web_sm")

# add domain-specific stopwords
domain_stops = {"said", "says", "say", "according", "reuters", "inc", "ltd"}
for w in domain_stops:
    nlp.Defaults.stop_words.add(w)
    # also set the lexeme attribute
    lex = nlp.vocab[w]
    lex.is_stop = True

# remove a word that you DO want to keep
nlp.Defaults.stop_words.discard("not")
nlp.vocab["not"].is_stop = False

# persist by saving the modified model
# nlp.to_disk("my_model_with_stops")

# Stopword removal HELPS:
#   - bag-of-words / TF-IDF (smaller vocab, less noise)
#   - topic modeling (LDA wants content words)
#   - search indexing (precision up, index smaller)

# Stopword removal HURTS:
#   - sentiment analysis ("not good" != "good")
#   - NER / entity extraction (alignment breaks)
#   - syntax parsing (function words are syntactic glue)
#   - transformer models (they were pretrained WITH stopwords)

from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained("bert-base-uncased")
# NEVER strip stopwords before feeding BERT — the model
# expects natural text and uses function words syntactically.

# rule: only remove stopwords for shallow features (BoW/TF-IDF),
# and only if you've checked it doesn't hurt your metric.

from sklearn.feature_extraction.text import CountVectorizer

docs = ["The cat sat on the mat.", "The dog sat on the log."]

# default: lowercase, word tokenizer (\b\w\w+\b), unigrams
vec = CountVectorizer()
X = vec.fit_transform(docs)
print(X.shape)            # (2, 6)  -> 2 docs, 6 unique words
print(vec.get_feature_names_out())
# ['cat' 'dog' 'log' 'mat' 'on' 'sat' 'the']

print(X.toarray())
# [[1 0 0 1 1 1 2]   'the' appears twice in doc 0
#  [0 1 1 0 1 1 2]]

# transform new text with the fitted vocabulary
new = vec.transform(["The cat and the dog."])

from sklearn.feature_extraction.text import CountVectorizer

docs = ["I love this movie", "I hate this movie"]

# unigrams + bigrams
vec = CountVectorizer(ngram_range=(1, 2))
X = vec.fit_transform(docs)
print(vec.get_feature_names_out())
# ['hate', 'hate this', 'love', 'love this', 'movie',
#  'this', 'this movie']

# bigrams capture phrases like "not good", "New York"
# but explode the vocabulary -> pair with min_df / max_features
vec2 = CountVectorizer(ngram_range=(2, 2), min_df=2)
# (2,2) = bigrams only; min_df=2 keeps bigrams seen >= 2 times

from sklearn.feature_extraction.text import HashingVectorizer

# stateless: no vocabulary kept, uses feature hashing
vec = HashingVectorizer(n_features=2**18, alternate_sign=False)
X = vec.transform(["doc one", "doc two"])
print(X.shape)   # (2, 262144)

# pros: no memory for vocabulary, streamable, fit_transform = transform
# cons: hashes are not interpretable, possible collisions,
#       no inverse_transform (can't map feature -> word)

# use for: very large / streaming corpora where you don't need
# to know which word a column corresponds to

from sklearn.feature_extraction.text import CountVectorizer

docs = ["a a a b b c rareword commontoken"] * 5

vec = CountVectorizer(
    min_df=2,          # ignore terms in <2 docs
    max_df=0.9,        # ignore terms in >90% of docs (corpus-specific stops)
    max_features=1000, # keep only top-1000 by frequency
    stop_words='english',
    token_pattern=r"\b[a-z]{2,}\b",  # only 2+ letter words
)
X = vec.fit_transform(docs)
print(vec.get_feature_names_out())

# fix a custom vocabulary (only these words are counted)
vec_fixed = CountVectorizer(vocabulary=["cat", "dog", "fish"])
print(vec_fixed.get_feature_names_out())

from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
from scipy.sparse import csr_matrix

docs = ["the cat sat", "the dog ran", "the cat ran"]
vec = CountVectorizer()
X = vec.fit_transform(docs)   # CSR sparse matrix

print(type(X))                # <class 'scipy.sparse.csr_matrix'>
print(X.shape, X.nnz)         # (3, 5) 9 non-zero entries

# NEVER call .toarray() on huge matrices — they explode in memory.
# operate on sparse directly:
col_sums = np.asarray(X.sum(axis=0)).ravel()   # document frequency
print(dict(zip(vec.get_feature_names_out(), col_sums)))

# find docs containing a word
word_idx = vec.vocabulary_["cat"]
rows_with_cat = X[:, word_idx].nonzero()[0]
print("docs with 'cat':", rows_with_cat)   # [0, 2]

from sklearn.feature_extraction.text import TfidfVectorizer

docs = ["the cat sat on the mat",
        "the dog sat on the log",
        "cats and dogs are pets"]

vec = TfidfVectorizer()        # default: L2-normalized rows
X = vec.fit_transform(docs)
print(vec.get_feature_names_out())

# tf-idf = term_freq * log(N / df) * (variations exist)
# down-weights words common across all docs ('the', 'sat')
print(X.toarray().round(3))

# each row is L2-normalized to length 1
import numpy as np
print(np.linalg.norm(X.toarray(), axis=1))   # [1. 1. 1.]

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.pipeline import Pipeline

# equivalent to TfidfVectorizer but in two steps
pipe = Pipeline([
    ("counts", CountVectorizer()),
    ("tfidf", TfidfTransformer(
        norm='l2',
        use_idf=True,
        smooth_idf=True,        # idf = ln((1+N)/(1+df)) + 1  (avoids div-by-zero)
        sublinear_tf=False,     # if True: tf = 1 + log(count)
    )),
])

# sublinear_tf compresses high counts:
#   count 5 -> tf 5 (linear) vs tf 1+log(5)≈2.6 (sublinear)
# useful when a word repeating 100x isn't 100x more important
pipe2 = Pipeline([
    ("counts", CountVectorizer()),
    ("tfidf", TfidfTransformer(sublinear_tf=True)),
])

from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np

docs = ["machine learning models", "deep learning networks",
        "natural language processing"]

vec = TfidfVectorizer()
X = vec.fit_transform(docs)
names = vec.get_feature_names_out()

# top features per document
for i, doc in enumerate(docs):
    row = X[i].toarray().ravel()
    top = np.argsort(row)[::-1][:3]
    print(f"doc {i}: {[(names[j], round(row[j], 3)) for j in top]}")
# doc 0: [('machine', 0.58), ('models', 0.58), ('learning', 0.41)]
# doc 1: [('deep', 0.5), ('networks', 0.5), ...]
# doc 2: [('natural', 0.41), ('processing', 0.41), ('language', 0.41)]

# inspect the learned idf weights
idf = dict(zip(names, vec.idf_))
print(sorted(idf.items(), key=lambda x: x[1])[:3])   # lowest idf = most common

from sklearn.feature_extraction.text import TfidfVectorizer

docs = ["not good at all", "very good indeed", "not bad actually"]

vec = TfidfVectorizer(
    stop_words='english',
    ngram_range=(1, 2),        # unigrams + bigrams
    min_df=1,
    sublinear_tf=True,
    max_features=5000,
)
X = vec.fit_transform(docs)
print(vec.get_feature_names_out())
# bigrams like 'not good', 'not bad' preserve negation

# this combination is a strong baseline for text classification
# with LogisticRegression or LinearSVC

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

corpus = [
    "Machine learning is fun",
    "Python is great for ML",
    "I love cooking pasta",
]
query = ["learning machine"]

vec = TfidfVectorizer()
X = vec.fit_transform(corpus)
q = vec.transform(query)

# cosine similarity between query and each doc
sims = cosine_similarity(q, X).ravel()
for doc, score in sorted(zip(corpus, sims), key=lambda x: -x[1]):
    print(f"{score:.3f}  {doc}")
# 0.580  Machine learning is fun
# 0.000  I love cooking pasta

# this is essentially a tiny search engine

from gensim.models import Word2Vec
from gensim.utils import simple_preprocess

sentences = [
    "the cat sat on the mat".split(),
    "the dog sat on the log".split(),
    "cats and dogs are pets".split(),
]
# simple_preprocess lowercases and tokenizes
sents = [simple_preprocess(" ".join(s)) for s in sentences]

model = Word2Vec(
    sentences=sents,
    vector_size=100,    # embedding dimension
    window=5,           # context window size
    min_count=1,        # ignore words with freq < min_count
    workers=4,
    sg=1,               # 1 = skip-gram, 0 = CBOW
    epochs=10,
)

print(model.wv["cat"].shape)            # (100,)
print(model.wv.most_similar("cat", topn=3))

from gensim.models import Word2Vec
from gensim.utils import simple_preprocess
from pathlib import Path

# read a large corpus line by line (don't load all into memory)
class SentenceIterator:
    def __init__(self, path):
        self.path = Path(path)
    def __iter__(self):
        for line in self.path.open(encoding="utf-8"):
            yield simple_preprocess(line)

sentences = SentenceIterator("corpus.txt")

model = Word2Vec(
    sentences,
    vector_size=200,
    window=8,
    min_count=5,         # drop very rare words
    workers=8,
    epochs=5,
    negative=10,         # negative samples per positive (skip-gram)
)
model.save("w2v.model")

# resume training later
# model = Word2Vec.load("w2v.model")
# model.train(more_sentences, total_examples=len(more), epochs=5)

from gensim.models import KeyedVectors

# load pretrained Google News vectors (3GB, 3M words)
# Download: https://code.google.com/archive/p/word2vec/
model = KeyedVectors.load_word2vec_format("GoogleNews-vectors-negative300.bin",
                                          binary=True)

# similarity
print(model.similarity("cat", "dog"))        # ~0.86
print(model.similarity("cat", "car"))        # ~0.28

# analogy: king - man + woman = queen
print(model.most_similar(positive=["king", "woman"],
                         negative=["man"], topn=1))
# [('queen', 0.71)]

# odd one out
print(model.doesnt_match("cat dog car horse".split()))   # 'car'

# which word doesn't fit
print(model.most_similar("python", topn=5))

from gensim.models import FastText

sentences = [
    "the cat sat on the mat".split(),
    "dogs and cats are friends".split(),
]

# FastText learns subword (character n-gram) embeddings
model = FastText(
    sentences,
    vector_size=100,
    window=5,
    min_count=1,
    min_n=3,            # min char n-gram
    max_n=6,            # max char n-gram
    epochs=10,
)

# KEY advantage: handles out-of-vocabulary words
print("catting" in model.wv)            # False (not seen in training)
print(model.wv["catting"].shape)        # (100,)  -> still gets a vector!
# vector is averaged from subword n-grams of 'catting'

print(model.wv.most_similar("cat", topn=3))

# GloVe: pretrained on Wikipedia+Gigaword
# Download: https://nlp.stanford.edu/projects/glove/
# Files: glove.6B.50d.txt, glove.6B.100d.txt, glove.6B.200d.txt, glove.6B.300d.txt

from gensim.scripts.glove2word2vec import glove2word2vec
from gensim.models import KeyedVectors

# convert GloVe format to word2vec format (one-time)
glove2word2vec("glove.6B.100d.txt", "glove.6B.100d.w2v.txt")

# load
glove = KeyedVectors.load_word2vec_format("glove.6B.100d.w2v.txt")
print(glove["king"].shape)        # (100,)
print(glove.most_similar("king", topn=3))

# fast alternative: load directly into a dict
import numpy as np
def load_glove(path, dim=100):
    embeddings = {}
    with open(path, encoding="utf-8") as f:
        for line in f:
            parts = line.split()
            embeddings[parts[0]] = np.asarray(parts[1:], dtype="float32")
    return embeddings

import numpy as np
import matplotlib.pyplot as plt
from sklearn.manifold import TSNE
from gensim.models import KeyedVectors

model = KeyedVectors.load_word2vec_format("glove.6B.100d.w2v.txt")

# pick a small set of related words
words = ["king", "queen", "man", "woman", "paris", "france",
         "london", "england", "dog", "cat", "horse", "cow"]
vectors = np.array([model[w] for w in words])

# reduce 100d -> 2d with t-SNE
tsne = TSNE(n_components=2, random_state=42, perplexity=5)
coords = tsne.fit_transform(vectors)

plt.figure(figsize=(8, 6))
for w, (x, y) in zip(words, coords):
    plt.scatter(x, y)
    plt.annotate(w, (x, y), xytext=(5, 5), textcoords="offset points")
plt.savefig("embeddings_tsne.png", dpi=120)
plt.show()

from transformers import AutoTokenizer, AutoModel
import torch

# load any pretrained model by name from the Hub
model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

inputs = tokenizer("Hello, BERT!", return_tensors="pt")
print(inputs.keys())   # input_ids, token_type_ids, attention_mask

with torch.no_grad():
    outputs = model(**inputs)

# last hidden state: (batch, seq_len, hidden_size)
print(outputs.last_hidden_state.shape)   # torch.Size([1, 6, 768])

# pooled output: [CLS] token passed through a linear+tanh
print(outputs.pooler_output.shape)        # torch.Size([1, 768])

from transformers import AutoTokenizer, AutoModel
import torch

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModel.from_pretrained("bert-base-uncased")

# contextual: 'bank' embeds differently in river vs money context
texts = ["I sat by the river bank.", "I deposited cash at the bank."]
for t in texts:
    inputs = tokenizer(t, return_tensors="pt")
    with torch.no_grad():
        out = model(**inputs)
    # find the index of 'bank'
    idx = (inputs["input_ids"][0] == tokenizer.convert_tokens_to_ids("bank")).nonzero()[0]
    vec = out.last_hidden_state[0, idx[0]]
    print(f"{t[:25]:25} bank-vec[:3] = {vec[:3].tolist()}")

# the two 'bank' vectors DIFFER -> this is the whole point of BERT
# (static embeddings like Word2Vec would give the same vector both times)

# pip install sentence-transformers
from sentence_transformers import SentenceTransformer, util

# models tuned to produce good sentence-level embeddings
model = SentenceTransformer("all-MiniLM-L6-v2")   # 384d, fast & strong

sentences = [
    "The cat sits on the mat.",
    "A feline is resting on a rug.",
    "Machine learning is fascinating.",
]
emb = model.encode(sentences, convert_to_tensor=True, normalize_embeddings=True)

# semantic similarity via cosine
print(util.cos_sim(emb, emb).round(3))
# [[1.    0.678 0.089]
#  [0.678 1.    0.107]
#  [0.089 0.107 1.   ]]

# semantic search: rank corpus by similarity to a query
query = model.encode("where is the cat?", convert_to_tensor=True)
hits = util.semantic_search(query, emb)[0]
print(hits)   # ranks 'cat sits on the mat' highest

from transformers import AutoTokenizer, AutoModel
import torch

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModel.from_pretrained("bert-base-uncased", output_hidden_states=True)

inputs = tokenizer("I love NLP", return_tensors="pt")
with torch.no_grad():
    out = model(**inputs)

# all 13 layers (embedding + 12 transformer layers)
hidden = out.hidden_states          # tuple of (1, seq, 768), len 13
print(len(hidden), hidden[0].shape)

# common pooling strategies for a sentence vector:
# 1. [CLS] token of the last layer
cls = hidden[-1][:, 0, :]
# 2. mean-pool over tokens (often better than [CLS])
mask = inputs["attention_mask"].unsqueeze(-1).float()
mean = (hidden[-1] * mask).sum(1) / mask.sum(1)
# 3. concat last 4 layers (rich feature, used in early BERT papers)
cat = torch.cat([hidden[-i][:, 0, :] for i in range(1, 5)], dim=-1)
print(cls.shape, mean.shape, cat.shape)   # all (1, 768) except cat (1, 3072)

import numpy as np
from gensim.models import KeyedVectors
from sentence_transformers import SentenceTransformer

# STATIC (Word2Vec/GloVe): same vector regardless of context
w2v = KeyedVectors.load_word2vec_format("glove.6B.100d.w2v.txt")
v1 = w2v["bank"]
v2 = w2v["bank"]
print("static cosine (always 1.0):", np.dot(v1, v2) / (np.linalg.norm(v1)*np.linalg.norm(v2)))

# CONTEXTUAL (BERT): vector depends on surrounding text
bert = SentenceTransformer("all-MiniLM-L6-v2")
from sklearn.metrics.pairwise import cosine_similarity
e_river = bert.encode("sat by the river bank")
e_money = bert.encode("deposited at the bank")
print("contextual cosine of two sentences:", cosine_similarity([e_river], [e_money])[0, 0].round(3))

# trade-offs:
#   static:   small, fast, fixed vocabulary, OOV-friendly only with FastText
#   contextual: large, slower, handles polysemy, needs GPU for speed

from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

# X: list of strings, y: labels
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
                                                    stratify=y, random_state=42)

clf = Pipeline([
    ("tfidf", TfidfVectorizer(ngram_range=(1, 2), min_df=2,
                              sublinear_tf=True, stop_words="english")),
    ("clf", LogisticRegression(max_iter=1000, class_weight="balanced")),
])

clf.fit(X_train, y_train)
print(classification_report(y_test, clf.predict(X_test)))

# predict on new text
print(clf.predict(["this is a great movie"]))   # ['positive']

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import Pipeline

clf = Pipeline([
    ("vec", CountVectorizer()),
    ("clf", MultinomialNB(alpha=1.0)),    # alpha = Laplace smoothing
])
clf.fit(X_train, y_train)

# MultinomialNB assumes word counts -> use CountVectorizer (not TF-IDF)
# alpha=1 (default) adds one to every count -> avoids zero probabilities
# lower alpha (0.1) for less smoothing when you have lots of data

# variants:
#   MultinomialNB - counts/TF-IDF (most common for text)
#   BernoulliNB   - binary features (word present/absent)
#   ComplementNB  - better on imbalanced text classes

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import LinearSVC
from sklearn.pipeline import Pipeline
from sklearn.calibration import CalibratedClassifierCV

# LinearSVC is much faster than SVC(kernel='linear') on sparse text
clf = Pipeline([
    ("tfidf", TfidfVectorizer(sublinear_tf=True)),
    ("clf", LinearSVC(C=1.0)),     # C = inverse regularization
])

# LinearSVC has no predict_proba by default; wrap if you need probabilities
calibrated = CalibratedClassifierCV(LinearSVC(), cv=3)
clf_prob = Pipeline([("tfidf", TfidfVectorizer()),
                     ("clf", calibrated)])

# tune C: smaller C = more regularization (helps with noisy text)
# LinearSVC is often the strongest shallow text classifier

from transformers import (AutoTokenizer, AutoModelForSequenceClassification,
                          TrainingArguments, Trainer)
import torch

model_name = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(
    model_name, num_labels=3
)

# tokenize the dataset
def tokenize(batch):
    return tokenizer(batch["text"], padding="max_length",
                     truncation=True, max_length=128)

train_ds = dataset["train"].map(tokenize, batched=True)
train_ds.set_format("torch", columns=["input_ids", "attention_mask", "label"])

args = TrainingArguments(
    output_dir="./clf",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    learning_rate=2e-5,
    eval_strategy="epoch",
)
trainer = Trainer(model=model, args=args,
                  train_dataset=train_ds, eval_dataset=dataset["test"])
trainer.train()

# multi-label: each document can have MULTIPLE tags
# (e.g. a news article is both 'politics' AND 'economy')

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.multiclass import OneVsRestClassifier
from sklearn.pipeline import Pipeline
from sklearn.metrics import f1_score

# y is a (n_samples, n_labels) binary matrix, NOT a 1-d array
clf = Pipeline([
    ("tfidf", TfidfVectorizer()),
    ("clf", OneVsRestClassifier(LogisticRegression(max_iter=1000))),
])
clf.fit(X_train, Y_train)         # Y_train: 2D binary array

# predict returns a 2D matrix
pred = clf.predict(X_test)
print(f1_score(Y_test, pred, average="micro"))

# alternatives:
#   MultiLabelBinarizer to convert list-of-labels -> 2D matrix
#   BinaryRelevance / ClassifierChains from scikit-multilearn
#   BERT with sigmoid + BCEWithLogitsLoss (one head per label)

from sklearn.metrics import (classification_report, confusion_matrix,
                              f1_score, precision_recall_fscore_support)

y_true = [0, 0, 1, 1, 2, 2]
y_pred = [0, 1, 1, 1, 2, 0]

# per-class precision / recall / F1
print(classification_report(y_true, y_pred, digits=3))

# F1 averaging strategies (multi-class):
#   macro  : average F1 over classes (treats all classes equally)
#   micro  : aggregate then compute (dominated by majority class)
#   weighted: average weighted by class support
print("macro F1:", f1_score(y_true, y_pred, average="macro"))
print("micro F1:", f1_score(y_true, y_pred, average="micro"))

# confusion matrix
print(confusion_matrix(y_true, y_pred))

# ROC-AUC needs probabilities; for multi-class use roc_auc_score with multi_class='ovr'

import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
nltk.download('vader_lexicon')

sia = SentimentIntensityAnalyzer()

for text in ["I love this!", "This is terrible :(",
             "It's okay, not great."]:
    scores = sia.polarity_scores(text)
    print(f"{text:25} {scores}")
# {'neg': 0.0, 'neu': 0.308, 'pos': 0.692, 'compound': 0.636}
# {'neg': 0.679, 'neu': 0.321, 'pos': 0.0, 'compound': -0.476}
# {'neg': 0.323, 'neu': 0.677, 'pos': 0.0, 'compound': -0.226}

# compound: normalized score in [-1, 1]
# common thresholds: >0.05 positive, <-0.05 negative, else neutral

from textblob import TextBlob   # pip install textblob

for text in ["I love NLP.", "This is the worst experience.",
             "The movie was okay."]:
    blob = TextBlob(text)
    print(f"{text:30} polarity={blob.sentiment.polarity:.2f} "
          f"subjectivity={blob.sentiment.subjectivity:.2f}")
# I love NLP.                    polarity=0.50 subjectivity=0.60
# This is the worst experience.  polarity=-0.60 subjectivity=0.75
# The movie was okay.            polarity=0.00 subjectivity=0.00

# polarity:     [-1, 1]  negative to positive
# subjectivity: [0, 1]   objective (fact) to subjective (opinion)

# also gives noun phrases, translation, etc.
print(blob.noun_phrases)

from transformers import pipeline

# zero-shot: pick any labels you want, no fine-tuning
clf = pipeline("sentiment-analysis",
               model="distilbert-base-uncased-finetuned-sst-2-english")
print(clf(["I love this product!", "Worst purchase ever."]))
# [{'label': 'POSITIVE', 'score': 0.9998}, {'label': 'POSITIVE'...}]

# zero-shot classification: works on ANY label set
zero = pipeline("zero-shot-classification",
                model="facebook/bart-large-mnli")
print(zero("The battery lasts only an hour.",
           candidate_labels=["positive", "negative", "neutral"]))
# {'labels': ['negative', 'neutral', 'positive'], 'scores': [0.95, ...]}

from transformers import (AutoTokenizer, AutoModelForSequenceClassification,
                          Trainer, TrainingArguments)
import evaluate

model_name = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

def tokenize(batch):
    return tokenizer(batch["text"], truncation=True, padding="max_length")

train_ds = dataset["train"].map(tokenize, batched=True)
train_ds = train_ds.rename_column("label", "labels")
train_ds.set_format("torch")

accuracy = evaluate.load("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    preds = logits.argmax(axis=-1)
    return accuracy.compute(predictions=preds, references=labels)

args = TrainingArguments("sentiment", num_train_epochs=3,
                         per_device_train_batch_size=16,
                         eval_strategy="epoch", learning_rate=2e-5)
trainer = Trainer(model=model, args=args, train_dataset=train_ds,
                  eval_dataset=dataset["test"], compute_metrics=compute_metrics)
trainer.train()

# Aspect-based sentiment: sentiment TOWARD a specific aspect
# "The food was great but the service was slow."
#   food -> positive, service -> negative

from transformers import pipeline

# approach 1: zero-shot on each aspect
zero = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
text = "The food was great but the service was slow."
for aspect in ["food", "service"]:
    result = zero(text, candidate_labels=["positive", "negative", "neutral"])
    print(f"{aspect}: {result['labels'][0]} ({result['scores'][0]:.2f})")

# approach 2: pyabsa library (specialized for ABSA)
# pip install pyabsa
# from pyabsa import AspectTermExtraction as ATE
# analyzer = ATE.AspectExtractor('english', auto_device=True)
# print(analyzer.predict(text))

# approach 3: prompt an LLM with aspect + text -> label

from gensim.corpora import Dictionary
from gensim.models import LdaModel

docs = [["cat", "dog", "pet"],
        ["dog", "bark", "pet"],
        ["python", "code", "program"],
        ["code", "bug", "python"]]

dictionary = Dictionary(docs)
corpus = [dictionary.doc2bow(d) for d in docs]   # bag-of-words

lda = LdaModel(
    corpus=corpus,
    id2word=dictionary,
    num_topics=2,
    passes=10,             # iterations over the corpus
    random_state=42,
    alpha="auto",          # document-topic prior
    eta="auto",            # topic-word prior
)

# top words per topic
for topic_id, words in lda.print_topics(num_words=4):
    print(f"Topic {topic_id}: {words}")

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

docs = ["cat dog pet", "dog bark pet", "python code program", "code bug python"]
vec = CountVectorizer(max_df=0.9, min_df=2, stop_words="english")
X = vec.fit_transform(docs)

lda = LatentDirichletAllocation(
    n_components=3,
    learning_method="batch",   # 'online' for large corpora
    max_iter=20,
    random_state=42,
)
lda.fit(X)

# top words per topic
words = vec.get_feature_names_out()
for i, topic in enumerate(lda.components_):
    top = [words[j] for j in topic.argsort()[:-6:-1]]
    print(f"Topic {i}: {top}")

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import NMF

docs = ["machine learning models", "deep learning networks",
        "natural language processing", "cooking recipes food"]

vec = TfidfVectorizer(max_features=1000, stop_words="english")
X = vec.fit_transform(docs)

nmf = NMF(n_components=2, random_state=42, l1_ratio=0.5, alpha_W=0.1)
W = nmf.fit_transform(X)   # document-topic
H = nmf.components_         # topic-word

words = vec.get_feature_names_out()
for i, topic in enumerate(H):
    print(f"Topic {i}: {[words[j] for j in topic.argsort()[:-6:-1]]}")
# NMF works with TF-IDF (LDA needs counts); topics are often cleaner
# on short documents

from gensim.models import CoherenceModel, LdaModel
from gensim.corpora import Dictionary

# compare LDA models with different numbers of topics
dictionary = Dictionary(docs)
corpus = [dictionary.doc2bow(d) for d in docs]

scores = []
for k in range(2, 11):
    lda = LdaModel(corpus=corpus, id2word=dictionary,
                   num_topics=k, passes=10, random_state=42)
    cm = CoherenceModel(model=lda, texts=docs,
                        dictionary=dictionary, coherence="c_v")
    score = cm.get_coherence()
    scores.append((k, score))
    print(f"k={k}: coherence={score:.3f}")

# pick the k with highest coherence (or where scores plateau)
# c_v in [0,1]; >0.5 is usually interpretable

# pip install pyldavis
import pyLDAvis
import pyLDAvis.gensim_models as gensimvis

# (assuming lda, corpus, dictionary from above)
vis = gensimvis.prepare(lda, corpus, dictionary)

# save as standalone HTML
pyLDAvis.save_html(vis, "lda_vis.html")
# open in a browser: bubbles = topics, sized by prevalence,
# positioned by inter-topic distance (PCoA on topic-word distances)

#pyLDAvis.enable_notebook()   # inline in Jupyter
#vis

# lambda slider: 1 = pure relevance by frequency,
# 0.6 = lift (rare-but-distinctive words rise)

# pip install bertopic
from bertopic import BERTopic
from sklearn.datasets import fetch_20newsgroups

docs = fetch_20newsgroups(subset="all")["data"][:1000]

# BERTopic: embeddings -> dimensionality reduction -> clustering
topic_model = BERTopic(
    language="english",
    calculate_probabilities=True,
    verbose=True,
)
topics, probs = topic_model.fit_transform(docs)

print(topic_model.get_topic_info().head())
print(topic_model.get_topic(0))   # top words for topic 0

# visualize
# topic_model.visualize_topics()
# topic_model.visualize_barchart()

# BERTopic finds the number of topics automatically (via HDBSCAN clusters)

import nltk
from nltk.util import ngrams
from collections import defaultdict, Counter

text = "the cat sat on the mat the dog sat on the log".split()
n = 2   # bigram model

# count bigram occurrences
model = defaultdict(Counter)
for w1, w2 in ngrams(text, n):
    model[w1][w2] += 1

# convert counts to probabilities
for w1 in model:
    total = sum(model[w1].values())
    for w2 in model[w1]:
        model[w1][w2] /= total

# generate next word given a context
import random
context = ("the",)
candidates = model[context[0]]
print(candidates)   # Counter({'cat': 0.25, 'dog': 0.25, 'mat': 0.25, 'log': 0.25})
print(random.choices(list(candidates), weights=candidates.values())[0])

import random
from collections import defaultdict

def build_chain(text, order=2):
    chain = defaultdict(list)
    words = text.split()
    for i in range(len(words) - order):
        state = tuple(words[i:i + order])
        chain[state].append(words[i + order])
    return chain

def generate(chain, length=30, seed=None):
    start = seed or random.choice(list(chain))
    out = list(start)
    state = start
    for _ in range(length):
        nxt = chain.get(state)
        if not nxt:
            break
        word = random.choice(nxt)
        out.append(word)
        state = tuple(out[-len(state):])
    return " ".join(out)

text = "the cat sat on the mat the cat ran the dog barked"
chain = build_chain(text, order=2)
print(generate(chain, length=20))

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_name = "gpt2"   # 'gpt2-medium', 'gpt2-large' for bigger
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)

prompt = "In a world where machines could think,"
input_ids = tokenizer.encode(prompt, return_tensors="pt")

# greedy / beam / sampling all live in generate()
output = model.generate(
    input_ids,
    max_new_tokens=80,
    do_sample=True,
    top_k=50,
    top_p=0.95,
    temperature=0.9,
    repetition_penalty=1.2,
    pad_token_id=tokenizer.eos_token_id,
)
print(tokenizer.decode(output[0], skip_special_tokens=True))

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelForCausalLM.from_pretrained("gpt2")
input_ids = tokenizer.encode("Once upon a time", return_tensors="pt")

# 1. GREEDY: always pick the argmax token. Deterministic, repetitive.
# 2. BEAM SEARCH: keep top-k sequences; deterministic, more coherent
#    but still bland.
beam = model.generate(input_ids, num_beams=5, max_new_tokens=50)

# 3. TEMPERATURE: sharpen (<1) or flatten (>1) the distribution.
#    temp=0.1 = nearly greedy; temp=1.5 = chaotic
sampler = model.generate(input_ids, do_sample=True, temperature=0.7,
                         top_k=0, max_new_tokens=50)

# 4. TOP-K: sample only from the k most likely tokens (k=50 typical)
topk = model.generate(input_ids, do_sample=True, top_k=50, max_new_tokens=50)

# 5. TOP-P (nucleus): sample from the smallest set whose cumulative
#    probability >= p (p=0.9 typical). Adapts to the distribution shape.
topp = model.generate(input_ids, do_sample=True, top_p=0.9, max_new_tokens=50)

from transformers import pipeline

generator = pipeline("text-generation", model="gpt2")

# zero-shot: just ask
zero = generator("Translate English to French: cheese ->", max_new_tokens=5)

# few-shot: show examples in the prompt, then ask for the next
prompt = """Translate English to French:
cheese -> fromage
bread -> pain
water ->"""
few = generator(prompt, max_new_tokens=3)

# instruction prompt (works best with instruction-tuned models)
instruction = """Summarize the following text in one sentence:
Text: The cat sat on the mat. It was a lazy afternoon...
Summary:"""
summary = generator(instruction, max_new_tokens=30)

# tips:
#   - be specific about format ("Output JSON", "Answer in one word")
#   - provide examples for non-obvious tasks
#   - chain-of-thought: "Let's think step by step."

# pip install googletrans==4.0.0-rc1
from googletrans import Translator

translator = Translator()

# single sentence
result = translator.translate("Hello, world!", src="en", dest="fr")
print(result.text)        # Bonjour le monde !
print(result.pronunciation)

# batch
results = translator.translate(
    ["Good morning", "Thank you"], src="en", dest="es")
for r in results:
    print(r.text)         # Buenos días / Gracias

# auto-detect source language
detected = translator.translate("Guten Tag")
print(detected.src)       # 'de'

# caveat: googletrans uses the free web API and may rate-limit / break.
# For production, use official Google / DeepL / Azure APIs.

from transformers import MarianTokenizer, MarianMTModel

# many language pairs available: 'Helsinki-NLP/opus-mt-{src}-{tgt}'
model_name = "Helsinki-NLP/opus-mt-en-fr"
tokenizer = MarianTokenizer.from_pretrained(model_name)
model = MarianMTModel.from_pretrained(model_name)

texts = ["Hello, how are you?", "I love natural language processing."]

batch = tokenizer(texts, return_tensors="pt", padding=True, truncation=True)
generated = model.generate(**batch, max_length=128, num_beams=4)
print(tokenizer.batch_decode(generated, skip_special_tokens=True))
# ['Bonjour, comment allez-vous ?', "J'aime le traitement du langage naturel."]

# Helsinki-NLP shipped hundreds of language pairs; for many-to-many use
# 'facebook/nllb-200-distilled-600M' (200 languages, one model)

from sacrebleu import corpus_bleu   # pip install sacrebleu

refs = [["The cat is on the mat."]]   # list of reference(s) per sentence
hyp = ["The cat is on the mat."]      # system output

bleu = corpus_bleu(hyp, refs)
print(bleu.score)    # 0-100 (0=worst, 100=perfect match)

# multiple references for one sentence
refs = [["The cat is on the mat.", "There is a cat on the mat."]]
hyp = ["A cat is on the mat."]
print(corpus_bleu(hyp, refs).score)

# BLEU = geometric mean of modified n-gram precision * brevity penalty
#   - precision: how many n-grams in hyp appear in references
#   - brevity penalty: penalizes too-short outputs

# caveats: BLEU ignores semantics, struggles on short sentences,
# correlates only moderately with human judgment

from transformers import AutoTokenizer

# translation models often use SentencePiece (BPE/unigram) tokenizers
tok = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de")

text = "Don't go there — it's 5€!"
enc = tok(text)
print(tok.convert_ids_to_tokens(enc["input_ids"]))
# ['▁Don', "'", 't', '▁go', '▁there', '▁—', '▁it', "'", 's', '▁5', '€', '!', '</s>']

# the leading '▁' (U+2581) marks a word start in SentencePiece.
# Strip it when reconstructing words:
pieces = [p.replace("▁", " ") for p in tok.convert_ids_to_tokens(enc["input_ids"])]
print("".join(pieces).strip())

# always use the model's own tokenizer, never a generic one —
# vocabulary and subword rules differ across translation models

from transformers import MarianMTModel, MarianTokenizer

# augment monolingual source data with back-translated paraphrases
src_model_name = "Helsinki-NLP/opus-mt-en-de"   # EN -> DE
bt_model_name = "Helsinki-NLP/opus-mt-de-en"     # DE -> EN (back)

mono_en = ["The weather is nice today.", "I am learning NLP."]

# forward: EN -> DE
fwd_tok = MarianTokenizer.from_pretrained(src_model_name)
fwd_model = MarianMTModel.from_pretrained(src_model_name)
de = fwd_model.generate(**fwd_tok(mono_en, return_tensors="pt", padding=True))

# back: DE -> EN (paraphrase of the original)
bt_tok = MarianTokenizer.from_pretrained(bt_model_name)
bt_model = MarianMTModel.from_pretrained(bt_model_name)
en_bt = bt_model.generate(**bt_tok(
    bt_tok.batch_decode(de, skip_special_tokens=True),
    return_tensors="pt", padding=True))

paraphrases = bt_tok.batch_decode(en_bt, skip_special_tokens=True)
print(paraphrases)   # ~same meaning, different wording

# Seq2Seq: an encoder maps the input sequence to a context vector,
# a decoder generates the output sequence from that context.
#
#   input -> [Encoder] -> context -> [Decoder] -> output
#
# Classic architecture (Sutskever et al. 2014):
#   - Encoder: RNN/LSTM/GRU processes input tokens, final hidden state
#     becomes the context vector.
#   - Decoder: another RNN initialized with the context, generates
#     output tokens one at a time.
#
# Used for: translation, summarization, QA, parsing, code generation.

import torch
import torch.nn as nn

class Encoder(nn.Module):
    def __init__(self, vocab_size, emb_dim, hid_dim):
        super().__init__()
        self.emb = nn.Embedding(vocab_size, emb_dim)
        self.rnn = nn.LSTM(emb_dim, hid_dim)
    def forward(self, src):
        embedded = self.emb(src)
        outputs, (hidden, cell) = self.rnn(embedded)
        return hidden, cell   # context

import torch
import torch.nn as nn

class Decoder(nn.Module):
    def __init__(self, vocab_size, emb_dim, hid_dim):
        super().__init__()
        self.emb = nn.Embedding(vocab_size, emb_dim)
        self.rnn = nn.LSTM(emb_dim, hid_dim)
        self.fc = nn.Linear(hid_dim, vocab_size)

    def forward(self, input_token, hidden, cell):
        # input_token: (1,) the previous output token
        emb = self.emb(input_token).unsqueeze(0)   # (1, 1, emb_dim)
        out, (hidden, cell) = self.rnn(emb, (hidden, cell))
        logits = self.fc(out.squeeze(0))            # (1, vocab_size)
        return logits, hidden, cell

# full generation loop (greedy decoding)
def generate(decoder, context_hidden, context_cell, sos_idx, eos_idx, max_len=20):
    hidden, cell = context_hidden, context_cell
    token = torch.tensor([sos_idx])
    output = []
    for _ in range(max_len):
        logits, hidden, cell = decoder(token, hidden, cell)
        token = logits.argmax(dim=-1)
        if token.item() == eos_idx:
            break
        output.append(token.item())
    return output

import torch
import torch.nn as nn

# During training, feed the GROUND TRUTH previous token (not the model's
# prediction) as the next input. This is 'teacher forcing'.

def train_step(encoder, decoder, src, trg, criterion, teacher_ratio=0.5):
    # src: (src_len, batch), trg: (trg_len, batch)
    hidden, cell = encoder(src)
    batch_size = src.size(1)
    trg_len = trg.size(0)
    vocab_size = decoder.fc.out_features

    outputs = torch.zeros(trg_len, batch_size, vocab_size)

    input_token = trg[0, :]   # <sos> token
    for t in range(1, trg_len):
        logits, hidden, cell = decoder(input_token, hidden, cell)
        outputs[t] = logits
        # teacher forcing: use real target as next input
        teacher_force = torch.rand(1).item() < teacher_ratio
        input_token = trg[t, :] if teacher_force else logits.argmax(dim=-1)

    return criterion(outputs[1:].reshape(-1, vocab_size),
                     trg[1:].reshape(-1))

import torch
import torch.nn as nn
import torch.nn.functional as F

# Bahdanau-style additive attention:
# the decoder looks at ALL encoder outputs (not just the final state)
# and learns which to focus on at each step.

class Attention(nn.Module):
    def __init__(self, hid_dim):
        super().__init__()
        self.attn = nn.Linear(hid_dim * 2, hid_dim)
        self.v = nn.Linear(hid_dim, 1, bias=False)

    def forward(self, hidden, encoder_outputs):
        # hidden: (1, batch, hid), encoder_outputs: (src_len, batch, hid)
        src_len = encoder_outputs.size(0)
        hidden_exp = hidden.repeat(src_len, 1, 1)
        energy = torch.tanh(self.attn(
            torch.cat((hidden_exp, encoder_outputs), dim=2)))
        attention = self.v(energy).squeeze(2)        # (src_len, batch)
        return F.softmax(attention, dim=0)           # weights over src

# context = sum(attention_weights * encoder_outputs)
# then concat context with decoder input embedding

# Greedy decoding picks the single most likely token each step.
# Beam search keeps the top-k candidate SEQUENCES, expanding each.

from transformers import AutoTokenizer, AutoModelForSeq2SeqLM

model = AutoModelForSeq2SeqLM.from_pretrained("Helsinki-NLP/opus-mt-en-fr")
tokenizer = AutoTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-fr")

input_ids = tokenizer("Hello, how are you?", return_tensors="pt").input_ids

# beam search: explore k hypotheses in parallel
output = model.generate(
    input_ids,
    num_beams=5,            # beam width
    max_length=50,
    length_penalty=0.6,     # <1 prefers shorter, >1 prefers longer
    early_stopping=True,
)
print(tokenizer.decode(output[0], skip_special_tokens=True))

# trade-off: larger beam = better quality but slower (O(k * steps))
# and a tendency toward generic / shorter outputs (length penalty helps)

# Self-attention lets each token look at every other token in the
# sequence to build a context-aware representation.
#
# For token i:  context_i = sum_j  attention(i, j) * value_j
#
# attention(i, j) = softmax( q_i . k_j / sqrt(d) )
#   q_i = query of token i  (what am I looking for?)
#   k_j = key of token j    (what do I contain?)
#   v_j = value of token j  (what info do I pass on?)
#
# Each token is projected to Q, K, V by learned weight matrices.

import torch
import torch.nn.functional as F
import math

def self_attention(x, W_q, W_k, W_v):
    # x: (seq_len, d_model)
    Q, K, V = x @ W_q, x @ W_k, x @ W_v
    d_k = K.size(-1)
    scores = Q @ K.transpose(-2, -1) / math.sqrt(d_k)
    weights = F.softmax(scores, dim=-1)
    return weights @ V   # (seq_len, d_model)

import torch
import torch.nn as nn
import math
import torch.nn.functional as F

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        assert d_model % n_heads == 0
        self.d_k = d_model // n_heads
        self.n_heads = n_heads
        self.q = nn.Linear(d_model, d_model)
        self.k = nn.Linear(d_model, d_model)
        self.v = nn.Linear(d_model, d_model)
        self.out = nn.Linear(d_model, d_model)

    def forward(self, x, mask=None):
        B, L, D = x.shape
        # project to (B, n_heads, L, d_k)
        Q = self.q(x).view(B, L, self.n_heads, self.d_k).transpose(1, 2)
        K = self.k(x).view(B, L, self.n_heads, self.d_k).transpose(1, 2)
        V = self.v(x).view(B, L, self.n_heads, self.d_k).transpose(1, 2)
        scores = (Q @ K.transpose(-2, -1)) / math.sqrt(self.d_k)
        if mask is not None:
            scores = scores.masked_fill(mask == 0, float("-inf"))
        attn = F.softmax(scores, dim=-1)
        ctx = attn @ V                          # (B, h, L, d_k)
        ctx = ctx.transpose(1, 2).contiguous().view(B, L, D)
        return self.out(ctx)

# The full Transformer block (Vaswani et al. 2017):
#
#   x -> MultiHeadAttention -> Add & LayerNorm -> FeedForward -> Add & LayerNorm
#   (Add = residual connection: out = x + Sublayer(x))
#
# Encoder: stacks N of these blocks (N=6 in base, 12 in BERT-base).
# Decoder: same but with masked self-attention + cross-attention
#          to the encoder output.
#
# Properties:
#   - no recurrence -> fully parallelizable, trains much faster than RNNs
#   - global receptive field from layer 1
#   - position-agnostic -> needs positional encoding
#   - quadratic memory in seq_len (O(L^2)) due to full attention

import torch.nn as nn

class TransformerBlock(nn.Module):
    def __init__(self, d_model=512, n_heads=8, d_ff=2048, dropout=0.1):
        super().__init__()
        self.attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout,
                                          batch_first=True)
        self.ff = nn.Sequential(
            nn.Linear(d_model, d_ff), nn.GELU(), nn.Linear(d_ff, d_model))
        self.norm1 = nn.LayerNorm(d_model)
        self.norm2 = nn.LayerNorm(d_model)
        self.drop = nn.Dropout(dropout)

    def forward(self, x, mask=None):
        a, _ = self.attn(x, x, x, attn_mask=mask, need_weights=False)
        x = self.norm1(x + self.drop(a))
        f = self.ff(x)
        x = self.norm2(x + self.drop(f))
        return x

import torch
import math
import torch.nn as nn

# Self-attention is permutation-invariant, so we ADD positional info
# to the embeddings so the model knows token order.

class SinusoidalPositionalEncoding(nn.Module):
    def __init__(self, d_model, max_len=5000):
        super().__init__()
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len).unsqueeze(1).float()
        div_term = torch.exp(
            torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.register_buffer("pe", pe.unsqueeze(0))   # (1, max_len, d_model)

    def forward(self, x):
        # x: (batch, seq_len, d_model)
        return x + self.pe[:, :x.size(1)]

# alternatives:
#   - learned positional embeddings (BERT, GPT): nn.Embedding(max_len, d_model)
#   - relative positional encoding (T5, Transformer-XL)
#   - RoPE (Rotary Position Embedding, used in LLaMA) -> handles long context

# Three Transformer families, three pretraining objectives:
#
# BERT (encoder-only, bidirectional):
#   - Masked Language Modeling (predict [MASK] tokens)
#   - Great for: classification, NER, QA (extractive), embeddings
#   - example: bert-base-uncased, RoBERTa, DistilBERT, DeBERTa
#
# GPT (decoder-only, autoregressive):
#   - Causal Language Modeling (predict next token)
#   - Great for: generation, chat, few-shot, code
#   - example: gpt2, gpt-neo, LLaMA, Mistral, Qwen
#
# T5 (encoder-decoder, text-to-text):
#   - Span corruption (mask spans, predict them)
#   - One model for EVERY task by framing as text->text
#   - example: t5-base, flan-t5, BART (similar, denoising autoencoder)

from transformers import AutoModel
bert = AutoModel.from_pretrained("bert-base-uncased")   # encoder
print(bert.config.model_type)   # bert

# pick the family that matches your task:
#   classification/NER -> encoder (BERT)
#   generation         -> decoder (GPT)
#   translation/summarization -> encoder-decoder (T5/BART)

from transformers import pipeline

# Extractive QA: pick a SPAN of the context as the answer.
qa = pipeline("question-answering",
              model="distilbert-base-cased-distilled-squad")

context = "Transformers were introduced in 2017. The model uses self-attention."
question = "When were transformers introduced?"

result = qa(question=question, context=context)
print(result)
# {'score': 0.97, 'start': 32, 'end': 36, 'answer': '2017'}

# start/end are character offsets into the context —
# you can highlight the answer in the original text:
print(context[:result['start']] + '[' + result['answer'] + ']' + context[result['end']:])

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_name = "google/flan-t5-base"   # instruction-tuned seq2seq
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)

prompt = """context: The Eiffel Tower is located in Paris and was built in 1889.
question: Where is the Eiffel Tower?
answer:"""

input_ids = tokenizer(prompt, return_tensors="pt").input_ids
out = model.generate(input_ids, max_new_tokens=20)
print(tokenizer.decode(out[0], skip_special_tokens=True))
# 'Paris'

# for open-ended QA without a fixed context, use a large LLM
# (Llama-2-Chat, Mistral-Instruct) and prompt with the question directly

# RAG: retrieve relevant documents, then let an LLM answer
# using those documents as context. Combines search + generation.

from sentence_transformers import SentenceTransformer, util

# 1. Build a vector index of your documents
docs = ["Paris is the capital of France.",
        "Python is a programming language.",
        "The Eiffel Tower was built in 1889."]
encoder = SentenceTransformer("all-MiniLM-L6-v2")
doc_emb = encoder.encode(docs, convert_to_tensor=True, normalize_embeddings=True)

# 2. Retrieve top-k for a query
query = encoder.encode("What is the capital of France?",
                       convert_to_tensor=True, normalize_embeddings=True)
hits = util.semantic_search(query, doc_emb, top_k=2)[0]
context = "\n".join(docs[h['corpus_id']] for h in hits)

# 3. Generate answer conditioned on retrieved context
prompt = f"Context:\n{context}\n\nQuestion: What is the capital of France?\nAnswer:"
# -> feed to an LLM; it reads the context and answers 'Paris'
print(prompt)

# SQuAD (Stanford Question Answering Dataset):
#   - 100k+ questions on Wikipedia paragraphs
#   - each answer is a SPAN in the paragraph
#   - the standard benchmark for extractive QA
#
# Load with Hugging Face datasets:
from datasets import load_dataset
squad = load_dataset("squad")
print(squad)
# DatasetDict({ train: 87599 rows, validation: 10570 rows })

ex = squad["train"][0]
print(ex.keys())
# dict_keys(['id', 'title', 'context', 'question', 'answers'])
print(ex["question"])      # 'Which NFL team represented the AFC...'
print(ex["answers"]["text"])   # ['Denver Broncos']  (sometimes multiple)

# SQuAD v2 adds unanswerable questions (model must abstain)
squad_v2 = load_dataset("squad_v2")

# Two standard metrics for extractive QA:
#   Exact Match (EM): 1 if prediction == gold answer exactly, else 0
#   F1: token-level F1 between prediction and gold (after normalization)

import re
from collections import Counter

def normalize(text):
    # SQuAD normalization: lowercase, remove articles/punct/extra space
    text = text.lower()
    text = re.sub(r"\b(a|an|the)\b", " ", text)
    text = " ".join(text.split())
    return re.sub(r"[^\w\s]", "", text)

def f1_score(pred, gold):
    p, g = normalize(pred).split(), normalize(gold).split()
    common = Counter(p) & Counter(g)
    n_common = sum(common.values())
    if n_common == 0:
        return 0.0
    precision = n_common / len(p)
    recall = n_common / len(g)
    return 2 * precision * recall / (precision + recall)

def exact_match(pred, gold):
    return float(normalize(pred) == normalize(gold))

print(f1_score("the cat", "a cat"))      # 1.0 (articles removed)
print(exact_match("Paris.", "paris"))    # 1.0 (case/punct normalized)

# Extractive: pick the most important SENTENCES from the source
# (no new text generated). TextRank ranks sentences by graph centrality.

import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def textrank_summary(text, n=3):
    sentences = text.split(". ")   # naive sentence split
    if len(sentences) <= n:
        return text
    # similarity graph: sentence vs sentence
    tfidf = TfidfVectorizer(stop_words="english").fit_transform(sentences)
    sim = cosine_similarity(tfidf)
    np.fill_diagonal(sim, 0)
    # PageRank-style power iteration
    scores = np.ones(len(sentences)) / len(sentences)
    for _ in range(50):
        scores = 0.85 * sim.dot(scores) / sim.sum(axis=0) + 0.15 / len(sentences)
    top = np.argsort(scores)[-n:]
    return ". ".join(sentences[i] for i in sorted(top))

text = "Sentence one. Sentence two is longer. Third sentence here. Fourth."
print(textrank_summary(text, n=2))

from gensim.summarization import summarize  # older gensim
# (removed in gensim 4.x; use 'sumy' or 'pytextrank' instead)

# modern alternative: sumy
# pip install sumy
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.text_rank import TextRankSummarizer

parser = PlaintextParser.from_string("Your long text here. Multiple "
    "sentences. The summarizer picks the most important ones.",
    Tokenizer("english"))
summarizer = TextRankSummarizer()
for sentence in summarizer(parser.document, sentences_count=2):
    print(sentence)

# sumy also offers: LuhnSummarizer, LsaSummarizer, LexRankSummarizer,
# EdmundsonSummarizer (heuristic-based)

from transformers import pipeline

# Abstractive: GENERATE a summary (may use words not in the source)
summarizer = pipeline("summarization",
                      model="facebook/bart-large-cnn")

article = """\
Scientists at CERN have announced the discovery of a new subatomic
particle. The finding, published in Nature, confirms a prediction made
decades ago and opens new avenues for research into dark matter. The
team analyzed data from the Large Hadron Collider over three years."""

summary = summarizer(article, max_length=50, min_length=20,
                     do_sample=False, truncation=True)
print(summary[0]["summary_text"])

# alternative models:
#   t5-base / flan-t5-base  (text-to-text, multitask)
#   google/pegasus-xsum     (very short, one-sentence summaries)

# ROUGE (Recall-Oriented Understudy for Gisting Evaluation):
# standard metric for summarization. Compares n-gram overlap.

# pip install rouge-score
from rouge_score import rouge_scorer

scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"],
                                  use_stemmer=True)
reference = "The cat sat on the mat."
hypothesis = "A cat is sitting on a mat."
scores = scorer.score(reference, hypothesis)
print(scores)
# rouge1: unigram overlap, rouge2: bigram, rougeL: longest common subsequence
# each gives (precision, recall, fmeasure)

# tips:
#   - ROUGE-1/ROUGE-2 F1 are the most reported
#   - ROUGE-L captures sentence-level order via LCS
#   - use multiple references for a fairer comparison
#   - ROUGE correlates with human judgment but ignores semantics

from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
import torch

# Problem: most summarizers have a 512/1024-token input limit.
# Long documents (papers, meetings) need special handling.

model_name = "google/long-t5-tglobal-base"   # handles long inputs
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSeq2SeqLM.from_pretrained(model_name)

# option 1: use a long-context model (LongT5, LED, BigBird)
inputs = tokenizer(long_text, return_tensors="pt", truncation=True,
                   max_length=4096)
out = model.generate(**inputs, max_length=256)

# option 2: chunk + summarize + combine (map-reduce)
def chunk_text(text, max_tokens=512):
    # split into overlapping windows, summarize each, then summarize
    # the concatenation of partial summaries
    pass

# option 3: hierarchical attention (summarize sections, then summarize
# section summaries)

from transformers import pipeline

# one-line inference for common tasks (downloads a default model)
clf = pipeline("sentiment-analysis")
print(clf("I love transformers!"))

qa = pipeline("question-answering")
print(qa(question="Who built BERT?", context="BERT was built by Google."))

ner = pipeline("ner", aggregation_strategy="simple")
print(ner("Barack Obama was born in Hawaii."))

summarizer = pipeline("summarization")
print(summarizer("Long text...", max_length=50))

generator = pipeline("text-generation", model="gpt2")
print(generator("Once upon a time", max_new_tokens=30))

# tasks available: sentiment-analysis, ner, question-answering,
# summarization, translation, text-generation, zero-shot-classification,
# fill-mask, feature-extraction, text2text-generation, image-classification...

from transformers import AutoTokenizer, AutoModel, AutoModelForSequenceClassification
import torch

model_name = "bert-base-uncased"

# Auto* classes pick the right implementation from the model's config
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)   # base encoder
clf_model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

inputs = tokenizer(["Hello", "World"], padding=True, return_tensors="pt")
with torch.no_grad():
    embeddings = model(**inputs).last_hidden_state   # (batch, seq, 768)

# task-specific Auto classes:
#   AutoModelForTokenClassification  (NER)
#   AutoModelForQuestionAnswering   (QA)
#   AutoModelForCausalLM             (GPT-style generation)
#   AutoModelForSeq2SeqLM            (T5/BART translation, summarization)
#   AutoModelForMaskedLM             (BERT-style MLM)

from transformers import (AutoTokenizer, AutoModelForSequenceClassification,
                          TrainingArguments, Trainer)
from datasets import load_dataset
import evaluate

model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

dataset = load_dataset("imdb")

def tokenize(batch):
    return tokenizer(batch["text"], truncation=True, padding="max_length", max_length=256)

train_ds = dataset["train"].map(tokenize, batched=True).rename_column("label", "labels")
train_ds.set_format("torch", columns=["input_ids", "attention_mask", "labels"])

accuracy = evaluate.load("accuracy")
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    return accuracy.compute(predictions=logits.argmax(-1), references=labels)

args = TrainingArguments(
    output_dir="imdb-bert",
    eval_strategy="epoch",
    save_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    num_train_epochs=3,
    load_best_model_at_end=True,
)
trainer = Trainer(model=model, args=args,
                  train_dataset=train_ds,
                  eval_dataset=dataset["test"].map(tokenize, batched=True),
                  compute_metrics=compute_metrics)
trainer.train()

from datasets import load_dataset, Dataset

# thousands of datasets, one API, streaming for huge ones
squad = load_dataset("squad", split="train[:1%]")
print(squad.column_names)   # ['id', 'title', 'context', 'question', 'answers']

# map over examples (batched for speed)
def preprocess(batch):
    return tokenizer(batch["context"], truncation=True)
squad = squad.map(preprocess, batched=True)

# filter / shuffle / select
squad = squad.filter(lambda ex: len(ex["context"]) > 100).shuffle(seed=42)

# create your own from a dict or CSV/JSON
ds = Dataset.from_dict({"text": ["a", "b"], "label": [0, 1]})
ds = Dataset.from_csv("data.csv")

# stream a huge dataset without downloading fully
wiki = load_dataset("wikipedia", "20220301.en", streaming=True)
for ex in wiki: break   # first example only

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer
from tokenizers.pre_tokenizers import Whitespace, Metaspace
from tokenizers.decoders import Metaspace as MetaspaceDecoder
from tokenizers.normalizers import NFD, Lowercase, StripAccents, Sequence

# build a tokenizer from scratch (fast, Rust-backed)
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.normalizer = Sequence([NFD(), Lowercase(), StripAccents()])
tokenizer.pre_tokenizer = Whitespace()

trainer = BpeTrainer(vocab_size=30000,
                     special_tokens=["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"])
tokenizer.train(files=["corpus.txt"], trainer=trainer)

# encode/decode
enc = tokenizer.encode("Hello, world!")
print(enc.ids, enc.tokens)
print(tokenizer.decode(enc.ids))

# save / load
tokenizer.save("my_tokenizer.json")
tok = Tokenizer.from_file("my_tokenizer.json")

# use with transformers: wrap as PreTrainedTokenizerFast

# The Hugging Face Hub hosts 100k+ models and datasets.
# Browse: https://huggingface.co/models

# load any model by its repo id
from transformers import AutoModel, AutoTokenizer
model = AutoModel.from_pretrained("bert-base-uncased")
# private/org models need an access token:
# AutoModel.from_pretrained("org/private-model", token="hf_xxx")

# share your own fine-tuned model
from huggingface_hub import HfApi
# 1. huggingface-cli login  (one-time, paste a token)
# 2. push from a Trainer:
trainer.push_to_hub("my-imdb-bert")
# 3. or push manually:
# api = HfApi()
# api.create_repo(repo_id="my-imdb-bert", repo_type="model")
# api.upload_folder(folder_path="imdb-bert", repo_id="my-imdb-bert")

# now anyone (or your team) can load it:
# AutoModel.from_pretrained("your-username/my-imdb-bert")