Hugging Face Transformers: The Ultimate Toolkit for State-of-the-Art NLP

from transformers import pipeline
 
# 1. Simple Pipeline API - NLP in one line
classifier = pipeline("sentiment-analysis")
result = classifier("Hugging Face Transformers is amazing!")
print(result)  # [{'label': 'POSITIVE', 'score': 0.999}]
 
# 2. Question Answering
qa_pipeline = pipeline("question-answering")
context = "Hugging Face was founded in 2016 in New York City."
question = "When was Hugging Face founded?"
answer = qa_pipeline(question=question, context=context)
print(answer)  # {'answer': '2016', 'score': 0.98}
 
# 3. Text Generation
generator = pipeline("text-generation", model="gpt2")
text = generator("The future of AI is", max_length=50)
print(text[0]['generated_text'])
 
# 4. Zero-shot Classification
classifier = pipeline("zero-shot-classification")
text = "This is a tutorial about Transformers library"
labels = ["education", "politics", "business", "technology"]
result = classifier(text, candidate_labels=labels)
print(result['labels'][0])  # 'technology'

from transformers import AutoTokenizer
 
# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
 
# Basic tokenization
text = "Hello, how are you doing today?"
tokens = tokenizer.tokenize(text)
print(tokens)  # ['hello', ',', 'how', 'are', 'you', 'doing', 'today', '?']
 
# Encoding and decoding
encoded = tokenizer.encode(text, return_tensors="pt")
print(encoded)  # tensor([[  101,  7592,  1010,  2129,  2024,  2017,  2725,  2651,  1029,   102]])
 
decoded = tokenizer.decode(encoded[0])
print(decoded)  # [CLS] hello, how are you doing today? [SEP]
 
# Batch encoding with padding and truncation
texts = [
    "Short text",
    "This is a much longer text that might need truncation",
    "Medium length text here"
]
 
batch_encoding = tokenizer(
    texts,
    padding=True,
    truncation=True,
    max_length=10,
    return_tensors="pt"
)
 
print(batch_encoding['input_ids'].shape)  # torch.Size([3, 10])

from transformers import AutoModel, AutoModelForSequenceClassification
import torch
 
# Load pre-trained model
model = AutoModel.from_pretrained("bert-base-uncased")
 
# Model architecture
print(model)
 
# Forward pass
with torch.no_grad():
    outputs = model(**batch_encoding)
    last_hidden_states = outputs.last_hidden_state
    print(f"Hidden states shape: {last_hidden_states.shape}")
 
# Task-specific models
classifier = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=3  # For 3-class classification
)
 
# Get predictions
outputs = classifier(**batch_encoding)
predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
print(f"Predictions shape: {predictions.shape}")

from transformers import AutoConfig
 
# Load and modify configuration
config = AutoConfig.from_pretrained("bert-base-uncased")
print(f"Hidden size: {config.hidden_size}")
print(f"Number of layers: {config.num_hidden_layers}")
print(f"Number of attention heads: {config.num_attention_heads}")
 
# Create custom configuration
config.num_hidden_layers = 6  # Smaller model
config.hidden_dropout_prob = 0.2  # More dropout
 
# Initialize model with custom config
custom_model = AutoModel.from_config(config)

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    TrainingArguments,
    Trainer,
    DataCollatorWithPadding
)
from datasets import load_dataset
import numpy as np
from sklearn.metrics import accuracy_score, f1_score
 
# Load dataset
dataset = load_dataset("imdb")
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
 
# Preprocessing function
def preprocess_function(examples):
    return tokenizer(
        examples["text"],
        truncation=True,
        padding=True,
        max_length=512
    )
 
# Tokenize dataset
tokenized_datasets = dataset.map(preprocess_function, batched=True)
 
# Data collator
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
 
# Load model
model = AutoModelForSequenceClassification.from_pretrained(
    "distilbert-base-uncased",
    num_labels=2
)
 
# Define metrics
def compute_metrics(eval_pred):
    predictions, labels = eval_pred
    predictions = np.argmax(predictions, axis=1)
    return {
        'accuracy': accuracy_score(labels, predictions),
        'f1': f1_score(labels, predictions, average='weighted')
    }
 
# Training arguments
training_args = TrainingArguments(
    output_dir="./results",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    push_to_hub=False,
)
 
# Initialize trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
    compute_metrics=compute_metrics,
)
 
# Train
trainer.train()
 
# Evaluate
results = trainer.evaluate()
print(f"Evaluation results: {results}")

from peft import get_peft_model, LoraConfig, TaskType
 
# Configure LoRA
peft_config = LoraConfig(
    task_type=TaskType.SEQ_CLS,
    inference_mode=False,
    r=8,
    lora_alpha=32,
    lora_dropout=0.1,
    target_modules=["query", "value"]
)
 
# Create PEFT model
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=2
)
peft_model = get_peft_model(model, peft_config)
 
# Print trainable parameters
peft_model.print_trainable_parameters()
# output: trainable params: 294912 || all params: 109514240 || trainable%: 0.269

from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch
 
# 8-bit quantization
quantization_config = BitsAndBytesConfig(
    load_in_8bit=True,
    bnb_8bit_compute_dtype=torch.float16
)
 
# Load quantized model
model = AutoModelForCausalLM.from_pretrained(
    "facebook/opt-6.7b",
    quantization_config=quantization_config,
    device_map="auto"
)
 
# 4-bit quantization for even smaller models
quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.float16,
    bnb_4bit_quant_type="nf4"
)

from transformers import VisionEncoderDecoderModel, ViTImageProcessor, AutoTokenizer
import torch
from PIL import Image
 
# Load vision-language model
model = VisionEncoderDecoderModel.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
feature_extractor = ViTImageProcessor.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
tokenizer = AutoTokenizer.from_pretrained("nlpconnect/vit-gpt2-image-captioning")
 
# Process image
image = Image.open("example.jpg")
pixel_values = feature_extractor(images=image, return_tensors="pt").pixel_values
 
# Generate caption
output_ids = model.generate(pixel_values, max_length=16, num_beams=4)
caption = tokenizer.decode(output_ids[0], skip_special_tokens=True)
print(f"Generated caption: {caption}")

from transformers import AutoModelForCausalLM, AutoTokenizer
import torch
 
model_name = "gpt2-medium"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name)
 
# Set pad token
tokenizer.pad_token = tokenizer.eos_token
 
def generate_text(prompt, max_length=100, temperature=0.8, top_p=0.9):
    inputs = tokenizer(prompt, return_tensors="pt", padding=True)
    
    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_length=max_length,
            temperature=temperature,
            top_p=top_p,
            do_sample=True,
            num_return_sequences=1,
            pad_token_id=tokenizer.eos_token_id
        )
    
    return tokenizer.decode(outputs[0], skip_special_tokens=True)
 
# Generate with different settings
creative_text = generate_text("The future of AI is", temperature=1.2)
focused_text = generate_text("The future of AI is", temperature=0.3)
 
print(f"Creative: {creative_text}")
print(f"Focused: {focused_text}")

from transformers import AutoModelForCausalLM, AutoTokenizer, Conversation, pipeline
 
# Load conversational model
chatbot = pipeline("conversational", model="microsoft/DialoGPT-medium")
 
# Create conversation
conversation = Conversation("Hello! How can I learn about transformers?")
conversation = chatbot(conversation)
print(conversation.messages[-1]["content"])
 
# Continue conversation
conversation.add_user_input("What are the best resources?")
conversation = chatbot(conversation)
print(conversation.messages[-1]["content"])

from transformers import AutoModel, AutoTokenizer
from transformers.onnx import export
from pathlib import Path
 
# Load model and tokenizer
model = AutoModel.from_pretrained("bert-base-uncased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
 
# Export to ONNX
onnx_path = Path("bert-base-uncased.onnx")
export(
    preprocessor=tokenizer,
    model=model,
    config=model.config,
    opset=13,
    output=onnx_path
)

from transformers import pipeline
from optimum.onnxruntime import ORTModelForSequenceClassification
 
# Load optimized model
model = ORTModelForSequenceClassification.from_pretrained(
    "optimum/distilbert-base-uncased-finetuned-sst-2-english"
)
tokenizer = AutoTokenizer.from_pretrained(
    "optimum/distilbert-base-uncased-finetuned-sst-2-english"
)
 
# Create optimized pipeline
classifier = pipeline("sentiment-analysis", model=model, tokenizer=tokenizer)
 
# Benchmark
import time
text = "This is amazing!" * 100
 
start = time.time()
for _ in range(100):
    result = classifier(text)
print(f"Inference time: {(time.time() - start) / 100:.3f}s per prediction")

# Clear GPU cache
import torch
torch.cuda.empty_cache()
 
# Use gradient checkpointing for large models
model.gradient_checkpointing_enable()
 
# Mixed precision training
from transformers import TrainingArguments
 
training_args = TrainingArguments(
    output_dir="./results",
    fp16=True,  # Enable mixed precision
    gradient_accumulation_steps=4,  # Accumulate gradients
    per_device_train_batch_size=8,
)

from transformers import DataCollatorForLanguageModeling
 
# Dynamic padding
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer,
    mlm=True,
    mlm_probability=0.15,
    pad_to_multiple_of=8  # Efficient for GPU
)
 
# Streaming large datasets
from datasets import load_dataset
 
dataset = load_dataset(
    "wikipedia",
    "20220301.en",
    streaming=True  # Don't load entire dataset
)

import gradio as gr
from transformers import pipeline
 
# Create pipeline
classifier = pipeline("sentiment-analysis")
 
# Create Gradio interface
def analyze_sentiment(text):
    result = classifier(text)[0]
    return f"{result['label']} (confidence: {result['score']:.2f})"
 
iface = gr.Interface(
    fn=analyze_sentiment,
    inputs=gr.Textbox(lines=3, placeholder="Enter text here..."),
    outputs="text",
    title="Sentiment Analysis",
    description="Analyze the sentiment of your text using Hugging Face Transformers"
)
 
iface.launch()

from transformers import TrainingArguments
 
training_args = TrainingArguments(
    output_dir="./results",
    report_to="wandb",  # Enable W&B logging
    run_name="bert-fine-tuning",
    logging_steps=10,
    eval_steps=100,
    save_strategy="epoch",
)