{ "categories": [ { "id": "precision-formats", "label": "Precision Formats", "order": 0, "entries": [ "bf16", "e2m1", "fp16", "fp32", "fp8", "int4", "int8", "nf4" ] }, { "id": "quantization-basics", "label": "Quantization Basics", "order": 1, "entries": [ "calibration-data", "context-window", "dequantization", "model-size-memory", "per-group-quantization", "ptq", "qat", "quantization", "token", "w4a16", "w8a8" ] }, { "id": "quantization-methods", "label": "Quantization Methods", "order": 2, "entries": [ "aqlm", "awq", "bitsandbytes", "eetq", "exl2", "gptq", "hqq", "imatrix", "kv-cache-quantization", "marlin", "quanto", "quip-sharp", "smoothquant", "spinquant", "unsloth-dynamic" ] }, { "id": "formats", "label": "Formats", "order": 3, "entries": [ "format-coreml", "format-ct2", "format-onnx", "format-openvino", "ggml", "gguf", "iq-quants", "k-quants", "mxfp4-moe", "safetensors" ] }, { "id": "attention-variants", "label": "Attention & Memory", "order": 4, "entries": [ "attention-mask", "cross-attention", "flash-attention", "gqa", "kv-cache", "linear-attention", "mha", "mla", "mqa", "paged-attention", "sdpa", "self-attention", "swa" ] }, { "id": "position-encodings", "label": "Position Encodings", "order": 5, "entries": [ "abf", "alibi", "learned-pe", "ntk-rope", "position-interpolation", "rope", "sinusoidal-pe", "yarn" ] }, { "id": "layer-types", "label": "Layer Types", "order": 6, "entries": [ "backbone", "clip", "diffusion-models", "ffn", "geglu", "layernorm", "mamba", "model-head", "ple", "pre-norm", "residual-connection", "rmsnorm", "rwkv", "seq2seq", "swiglu", "vit", "vlm", "weight-tying" ] }, { "id": "scaling-patterns", "label": "Scaling & Serving", "order": 7, "entries": [ "api-provider", "chain-of-thought", "chatbot-arena", "chunked-prefill", "context-engineering", "continuous-batching", "data-contamination", "data-parallelism", "dense-models", "evals", "expert-parallelism", "expert-routing", "gpqa", "gpu-memory", "gsm8k", "humaneval", "inference-engine", "inference-metrics", "inference", "knowledge-distillation", "latency", "llm-as-judge", "llmops", "metr", "mmlu", "model-merging", "moe", "mt-bench", "observability", "perplexity", "prefix-caching", "pruning", "rag", "reasoning-models", "red-teaming", "rouge", "scaling-test-time", "speculative-decoding", "test-time-compute", "thinking-tokens", "tp-pp", "truthfulqa", "zero" ] }, { "id": "training-pipeline", "label": "Training Pipeline", "order": 8, "entries": [ "causal-lm", "cpt", "dpo", "gradient-checkpointing", "grpo", "kto", "mixed-precision-training", "orpo", "post-training", "pre-training", "rlhf", "rlvr", "scaling-laws", "sft", "simpo", "training-recipe", "transfer-learning" ] }, { "id": "sampling-decoding", "label": "Sampling & Decoding", "order": 8, "entries": [ "beam-search", "greedy-decoding", "min-p", "repetition-penalty", "structured-output", "temperature", "top-k", "top-p" ] }, { "id": "fine-tuning-methods", "label": "Fine-Tuning Methods", "order": 9, "entries": [ "adapters", "dora", "fine-tuning", "full-ft", "ia3", "lora", "peft", "qlora" ] }, { "id": "model-naming", "label": "Model Naming", "order": 10, "entries": [ "chat-template", "license-types", "model-card", "open-weights", "size-A", "size-B", "size-E", "size-T", "size-context", "size-x", "tag-alignment", "tag-base", "tag-coder", "tag-embed", "tag-ft", "tag-guard", "tag-hf", "tag-instruct", "tag-long", "tag-math", "tag-merged", "tag-moe-suffix", "tag-mrl", "tag-reward", "tag-scale", "tag-scout-maverick", "tag-vision" ] }, { "id": "hf-organizations", "label": "HF Organizations", "order": 11, "entries": [ "huggingface-hub", "olmo", "org-bartowski", "org-deepseek", "org-google", "org-meta", "org-microsoft", "org-mistral", "org-mlx-community", "org-mradermacher", "org-nousresearch", "org-qwen", "org-thebloke", "org-turboderp" ] }, { "id": "serving-tools", "label": "Serving Tools", "order": 12, "entries": [ "inspect-eval", "tool-accelerate", "tool-crewai", "tool-dask", "tool-exllamav2", "tool-langchain", "tool-langfuse", "tool-langsmith", "tool-llamacpp", "tool-llamaindex", "tool-lmstudio", "tool-n8n", "tool-ollama", "tool-outlines", "tool-ray", "tool-sglang", "tool-tensorrt-llm", "tool-tgi", "tool-triton", "tool-vllm" ] }, { "id": "tokenizers", "label": "Tokenizers", "order": 13, "entries": [ "tokenizer-bpe", "tokenizer-spm", "tokenizer-tiktoken", "tokenizer-wordpiece" ] }, { "id": "datasets-recipes", "label": "Datasets & Recipes", "order": 14, "entries": [ "data-deduplication", "data-mixing", "dataset-dolphin", "dataset-evolinstruct", "dataset-hermes", "dataset-orca", "dataset-platypus", "dataset-ultrachat", "synthetic-data" ] }, { "id": "agents-and-tools", "label": "Agents & Tool Use", "order": 15, "entries": [ "a2a", "agentic-ai", "dspy", "function-calling", "guardrails", "mcp", "pydantic-ai", "react" ] }, { "id": "safety-alignment", "label": "Safety & Alignment", "order": 16, "entries": [ "constitutional-ai", "hallucination", "jailbreak", "prompt-injection", "reward-model", "rlaif" ] }, { "id": "prompting", "label": "Prompting", "order": 17, "entries": [ "few-shot", "prompt-engineering", "system-prompt", "zero-shot" ] }, { "id": "embeddings-retrieval", "label": "Embeddings & Retrieval", "order": 18, "entries": [ "embeddings", "faiss", "vector-database" ] } ], "entries": { "a2a": { "id": "a2a", "name": "A2A Protocol", "expansion": "Agent-to-Agent Protocol (Google)", "category": "agents-and-tools", "oneliner": "Google's open protocol for AI agents to discover each other's capabilities and collaborate on tasks across different frameworks and vendors.", "explanation": "The Agent-to-Agent (A2A) protocol is an open standard from Google for enabling AI agents built on different frameworks to communicate and collaborate. Each agent publishes an Agent Card describing its capabilities, and agents can delegate tasks to each other through a standardized JSON-RPC interface. A2A complements Anthropic's MCP — while MCP connects agents to tools and data sources, A2A connects agents to other agents.", "fundamentals": "Core concepts: Agent Card (JSON capability description published at /.well-known/agent.json), Task (unit of work with lifecycle: submitted → working → completed/failed), Message (communication between agents within a task), Artifact (output produced by a task). Transport: HTTP + JSON-RPC 2.0, with optional SSE for streaming. Discovery: clients fetch Agent Cards to understand what an agent can do before sending tasks. Designed for interop across LangChain, CrewAI, Vertex AI, and other frameworks.", "related": [ "mcp", "agentic-ai", "function-calling" ], "seen_in": [ "documentation" ], "foundational_papers": [], "resources": [ { "label": "A2A Protocol spec", "url": "https://github.com/google/A2A" }, { "label": "Google A2A announcement", "url": "https://developers.googleblog.com/en/a2a-a-new-era-of-agent-interoperability/" } ], "confidence": "high", "verified_date": "2026-04-11" }, "abf": { "id": "abf", "name": "ABF", "expansion": "Adjusted Base Frequency", "category": "position-encodings", "oneliner": "Simplest long-context approach: increase RoPE base from 10,000 to 500K-1M during training. No post-hoc scaling needed.", "explanation": "Adjusted Base Frequency extends a model's context window by increasing the base constant in the RoPE position encoding formula from the default 10,000 to a much larger value. Longer wavelengths let the model distinguish positions over longer sequences without confusion, while local token discrimination is preserved. This simple change requires continued pre-training. Llama 3 uses a base of 500,000 and Qwen 2 uses 1,000,000.", "fundamentals": "$\\theta_i$ = B^(-2i/d). Max wavelength = 2$\\pi$·B. base=10K → ~63K positions; base=500K → ~3.1M positions; base=1M → ~6.3M positions. High-freq (i=0): $\\theta_0$=1 always (base^0=1) — local info preserved for any base. Mathematically equivalent to NTK-aware with specific $\\alpha$, but applied during training not post-hoc. The model learns to use the extended position range natively — no distributional mismatch.", "seen_in": [ "model-config" ], "related": [ "ntk-rope", "position-interpolation", "pre-training", "rope", "token", "yarn" ], "sources": [ "Meta, 'The Llama 3 Herd of Models,' 2024, arXiv:2407.21783", "Qwen Team, 'Qwen2 Technical Report,' 2024, arXiv:2407.10671", "Xiong et al., 'Effective Long-Context Scaling,' 2023, arXiv:2309.16039" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "Llama 3 paper (rope_theta=500K)", "url": "https://arxiv.org/abs/2407.21783" }, { "label": "Qwen2 Technical Report (rope_theta=1M)", "url": "https://arxiv.org/abs/2407.10671" } ] }, "adapters": { "id": "adapters", "name": "Adapters (General Concept)", "expansion": "Adapter Modules — the broad category of injected trainable modules", "category": "fine-tuning-methods", "oneliner": "Small trainable modules inserted into frozen models. Houlsby (serial bottleneck), Pfeiffer (efficient), parallel, LoRA are all adapter variants.", "explanation": "Adapters are small trainable modules inserted into the layers of a frozen pre-trained model to teach it new tasks without updating all its weights. They dramatically reduce the cost of fine-tuning because only the adapter parameters are trained while the original model stays intact. Variants include Houlsby bottleneck adapters, Pfeiffer single-layer adapters, parallel adapters, and LoRA.", "fundamentals": "Houlsby: adapter(x) = x + W_up·$\\sigma$(W_down·x). 4dr params/layer. Pfeiffer: 2dr/layer. LoRA: 2dr per target module. Inference: Houlsby 5-10% latency. LoRA merged: zero.", "seen_in": [ "model-name", "filename" ], "related": [ "ia3", "inference", "lora", "peft" ], "foundational_papers": [ { "title": "Parameter-Efficient Transfer Learning for NLP", "authors": "Houlsby et al.", "venue": "ICML 2019", "arxiv": "1902.00751" }, { "title": "Towards a Unified View of Parameter-Efficient Transfer Learning", "authors": "He et al.", "venue": "ICLR 2022", "arxiv": "2110.04366" }, { "title": "AdapterFusion: Non-Destructive Task Composition for Transfer Learning", "authors": "Pfeiffer et al.", "venue": "EACL 2021", "arxiv": "2005.00247" } ], "sources": [ "Houlsby et al., arXiv:1902.00751" ], "confidence": "high", "verified_date": "2026-04-10" }, "agentic-ai": { "id": "agentic-ai", "name": "Agentic AI", "expansion": "Agentic Artificial Intelligence", "category": "agents-and-tools", "oneliner": "AI systems that plan, use tools, retain memory, and execute multi-step tasks autonomously. The paradigm shift from prompt-response to goal-driven agents.", "explanation": "Agentic AI is the paradigm in which a language model serves as a reasoning engine inside an orchestration loop that can plan actions, invoke external tools, observe results, and iterate until a goal is met. Unlike a vanilla chatbot that produces a single reply, an agentic system breaks a request into sub-tasks, calls APIs or databases, writes and runs code, and self-corrects based on feedback.", "fundamentals": "Core loop: Observe -> Think -> Act -> Observe. The LLM receives the goal plus accumulated context (scratchpad), reasons about the next step, emits a structured tool call, the harness executes it, appends the result, and re-prompts. ReAct (Reason + Act) is the foundational pattern. Key design axes: single-agent vs. multi-agent, sequential vs. parallel tool calls, human-in-the-loop checkpoints, and memory persistence (short-term scratchpad vs. long-term vector store). Reliability depends on the model's instruction-following and the harness's error recovery.", "related": [ "function-calling", "mcp", "rag", "inference-engine" ], "seen_in": [ "documentation", "serving-config" ], "foundational_papers": [ { "title": "ReAct: Synergizing Reasoning and Acting in Language Models", "authors": "Yao et al.", "venue": "ICLR 2023", "arxiv": "2210.03629" }, { "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "authors": "Schick et al.", "venue": "NeurIPS 2023", "arxiv": "2302.04761" } ], "resources": [ { "label": "Agentic AI explained (MIT Sloan)", "url": "https://mitsloan.mit.edu/ideas-made-to-matter/agentic-ai-explained" }, { "label": "What is Agentic AI (IBM)", "url": "https://www.ibm.com/think/topics/agentic-ai" } ], "confidence": "high", "verified_date": "2026-04-11" }, "alibi": { "id": "alibi", "name": "ALiBi", "expansion": "Attention with Linear Biases", "category": "position-encodings", "oneliner": "Adds a fixed linear penalty to attention scores based on query-key distance — no position embeddings at all, with good length extrapolation.", "explanation": "ALiBi is a position encoding method that adds a fixed distance-based penalty directly to attention scores instead of using position embeddings. Nearby tokens receive a small penalty while distant tokens receive a larger one, creating a natural decay of attention with distance. This simple approach allows models to extrapolate to longer sequences than they were trained on. ALiBi is used in BLOOM, Falcon, and MPT, but has become less common as RoPE scaling methods improved.", "fundamentals": "Modified attention: a_{i,j} = $q_i^T$·$k_j$ - m_h·|i-j|. For H=8 heads: slopes = {1/2, 1/4, 1/8, ..., 1/256}. Steep slopes = strong locality, gentle slopes = far attention. Causal: bias = -m_h·(i-j) for j≤i. The bias matrix is lower-triangular. $O(1)$ additional params (fixed slopes). Extrapolation works because linear function is defined for any distance — no OOD issue. Cannot learn to attend strongly to a specific far position through positional info alone (content can override).", "seen_in": [ "model-config" ], "related": [ "rope", "sinusoidal-pe", "learned-pe" ], "sources": [ "Press et al., 'Train Short, Test Long,' ICLR 2022, arXiv:2108.12409" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "Train Short, Test Long: Attention with Linear Biases Enables Input Length Generalization", "authors": "Press et al.", "venue": "ICLR 2022", "arxiv": "2108.12409" } ] }, "api-provider": { "id": "api-provider", "name": "API Providers", "expansion": "LLM API Providers", "category": "scaling-patterns", "oneliner": "Companies offering hosted LLM inference via API — OpenAI, Anthropic, Google, Cohere, Mistral, Together AI, Groq, Fireworks, and others.", "explanation": "API providers host language models and offer them as a service via HTTP endpoints, so you can use powerful models without managing GPU infrastructure. OpenAI (GPT-4, o1), Anthropic (Claude), and Google (Gemini) offer proprietary models. Together AI, Fireworks, and Groq offer open-weight models (Llama, Mistral, Qwen) at competitive prices with high throughput. Groq is notable for custom LPU hardware delivering extremely low latency.", "related": [ "inference", "inference-engine" ], "seen_in": [ "documentation" ], "foundational_papers": [], "resources": [ { "label": "OpenAI API", "url": "https://platform.openai.com" }, { "label": "Together AI", "url": "https://together.ai" } ], "confidence": "high", "verified_date": "2026-04-11" }, "aqlm": { "id": "aqlm", "name": "AQLM", "expansion": "Additive Quantization of Language Models", "category": "quantization-methods", "oneliner": "Extreme compression (2-bit) using learned additive codebooks — highest quality at very low bitwidths.", "explanation": "AQLM is a weight compression method that represents groups of model weights as combinations of entries from multiple small learned codebooks. Each weight group is encoded as a sum of codewords drawn from separate codebooks, a technique called additive vector quantization. This approach dominates at extreme 2-bit compression where simpler rounding methods fall apart. Quantization is slow but inference runs efficiently with custom GPU kernels.", "fundamentals": "For a group of K weights: w $\\approx$ c1[i1] + c2[i2] + ... + cM[iM] where c_m are learned codebooks and i_m are indices stored as compact integers. Training alternates: fix codebooks, find best indices; fix indices, optimize codebooks. Effective bpw depends on M, B, K parameters.", "seen_in": [ "model-name", "repo-name" ], "related": [ "inference", "quantization", "quip-sharp" ], "sources": [ "Egiazarian et al., 'AQLM,' arXiv:2401.06118" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "AQLM: Extreme Compression of Language Models via Additive Quantization", "authors": "Egiazarian et al.", "venue": "ICML 2024", "arxiv": "2401.06118" } ] }, "attention-mask": { "id": "attention-mask", "name": "Attention Mask", "expansion": "Attention Mask", "category": "attention-variants", "oneliner": "A binary mask that controls which token pairs can attend to each other, handling padding, causality, and custom patterns.", "explanation": "An attention mask is a binary tensor that controls which token positions can attend to each other during the attention computation. It serves two main purposes: padding masks prevent the model from attending to placeholder tokens when batching sequences of different lengths, and causal masks enforce the left-to-right constraint in autoregressive generation. In HuggingFace Transformers, real tokens are marked with 1 and padding with 0.", "fundamentals": "Applied as: $\\text{Attention}(Q,K,V) = \\text{softmax}\\left(\\frac{QK^\\top}{\\sqrt{d_k}} + M_{\\text{add}}\\right)V$, where $M_{\\text{add}}$ contains $0$ for allowed positions and $-\\infty$ for blocked positions. For padded batches, the padding mask has shape $1 \\times L$ and is broadcast to $L \\times L$, combined with the causal mask via element-wise minimum.", "related": [ "mha", "token" ], "seen_in": [ "code", "documentation" ], "foundational_papers": [ { "title": "Attention Is All You Need", "authors": "Vaswani et al.", "venue": "NeurIPS 2017", "arxiv": "1706.03762" } ], "resources": [ { "label": "HuggingFace Glossary: Attention mask", "url": "https://huggingface.co/docs/transformers/glossary#attention-mask" } ], "confidence": "high", "verified_date": "2026-04-11" }, "awq": { "id": "awq", "name": "AWQ", "expansion": "Activation-Aware Weight Quantization", "category": "quantization-methods", "oneliner": "Identifies the most important ~1% of weight channels via activation magnitudes, then protects them with per-channel scaling.", "explanation": "AWQ is a weight quantization method that identifies which weight channels matter most by looking at activation patterns during inference. Channels that consistently produce large activations amplify any rounding errors, so AWQ scales those channels before quantizing to protect them. It only needs a quick forward pass for calibration with no gradients or Hessian, making it fast even for 70B models.", "fundamentals": "1) Calibrate: s_x(j) = mean(|X[:,j]|) per channel j. 2) Compute scaling: s(j) = s_x(j)^$\\alpha$, $\\alpha$ found by grid search minimizing $\\|\\|Q(W·diag(s))·(X/diag(s)) - WX\\|\\|$. 3) Store W' = W·diag(s), quantize W'. 4) At runtime: activations divided by s (fused into preceding LayerNorm).", "seen_in": [ "model-name", "filename" ], "related": [ "gptq", "inference", "layernorm", "quantization" ], "sources": [ "Lin et al., 'AWQ,' MLSys 2024, arXiv:2306.00978" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration", "authors": "Lin et al.", "venue": "MLSys 2024", "arxiv": "2306.00978" } ], "resources": [ { "label": "HuggingFace — Selecting a Quantization Method", "url": "https://huggingface.co/docs/transformers/quantization/selecting" } ] }, "backbone": { "id": "backbone", "name": "Backbone", "expansion": "Model Backbone (trunk)", "category": "layer-types", "oneliner": "The main body of a model — embedding layer plus transformer blocks — that produces contextual representations before any task-specific head.", "explanation": "The backbone is the main body of a transformer model, comprising the embedding table, positional encoding, and the full stack of transformer layers. It converts a sequence of token IDs into rich contextual hidden states that a task-specific head then uses to make predictions. When people reference a model like LLaMA-3 70B, they mostly mean the backbone. The backbone is pretrained once and can be reused across many tasks via different heads or fine-tuning.", "fundamentals": "A decoder-only backbone maps tokens $x_1, \\ldots, x_n$ to hidden states through $L$ layers. Each layer: $h' = h + \\text{Attention}(\\text{Norm}(h))$, then $h'' = h' + \\text{FFN}(\\text{Norm}(h'))$. Total params $\\approx L \\cdot 12d^2$ for standard MHA with $4d$ FFN, plus embedding $|V| \\cdot d$.", "related": [ "ffn", "mha", "model-head", "residual-connection", "token" ], "seen_in": [ "model-config", "documentation" ], "foundational_papers": [ { "title": "Attention Is All You Need", "authors": "Vaswani et al.", "venue": "NeurIPS 2017", "arxiv": "1706.03762" } ], "resources": [ { "label": "HuggingFace Model summary", "url": "https://huggingface.co/docs/transformers/model_summary" }, { "label": "Sebastian Raschka — The Big LLM Architecture Comparison", "url": "https://magazine.sebastianraschka.com/p/the-big-llm-architecture-comparison" } ], "confidence": "high", "verified_date": "2026-04-11" }, "beam-search": { "id": "beam-search", "name": "Beam Search", "expansion": "Beam Search Decoding", "category": "sampling-decoding", "oneliner": "A decoding algorithm that tracks the top-k most probable partial sequences at each step, finding higher-quality outputs than greedy decoding at the cost of more compute.", "explanation": "Beam search maintains a set of k candidate sequences (the beam) at each generation step, expanding each by one token and keeping only the k most probable paths. This explores a wider search space than greedy decoding, often finding better overall sequences. It was the standard decoding method for machine translation before sampling became popular for chat.", "related": [ "greedy-decoding", "temperature", "top-p" ], "seen_in": [ "code", "documentation" ], "foundational_papers": [], "resources": [ { "label": "HuggingFace — Decoding strategies", "url": "https://huggingface.co/blog/mlabonne/decoding-strategies" } ], "confidence": "high", "verified_date": "2026-04-11" }, "bf16": { "id": "bf16", "name": "bf16 / bfloat16", "expansion": "Brain Floating Point 16-bit (Google Brain Float 16)", "category": "precision-formats", "oneliner": "A 16-bit float with fp32's exponent range but reduced mantissa — designed for deep learning training without loss scaling.", "explanation": "BFloat16 is a 16-bit floating-point format developed by Google Brain that keeps the same 8-bit exponent as 32-bit float, giving it an enormous numerical range. This means training gradients almost never overflow and loss scaling is unnecessary, unlike with fp16. The tradeoff is a shorter mantissa giving less precision, but neural networks tolerate this well. BFloat16 is the default training format for most major labs and runs on TPUs, NVIDIA Ampere GPUs, and newer.", "fundamentals": "Bit layout: 1 sign | 8 exponent (bias 127) | 7 mantissa. Same exponent as fp32 — conversion is just truncating the lower 16 bits. Precision: ~2.4 decimal digits (vs fp16's ~3.3). Two values differing by less than 1 part in 128 map to the same bf16.", "seen_in": [ "model-name", "filename", "training-config" ], "related": [ "fp16", "fp32" ], "sources": [ "Kalamkar et al., 'A Study of BFLOAT16 for Deep Learning Training,' arXiv:1905.12322" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "A Study of BFLOAT16 for Deep Learning Training", "authors": "Kalamkar et al.", "venue": "2019", "arxiv": "1905.12322" } ] }, "bitsandbytes": { "id": "bitsandbytes", "name": "BitsAndBytes (bnb)", "expansion": "bitsandbytes — Tim Dettmers' quantization library", "category": "quantization-methods", "oneliner": "The library behind LLM.int8() and QLoRA. Offers 8-bit (absmax) and 4-bit (nf4/fp4) quantization with deep HuggingFace integration.", "explanation": "Bitsandbytes is a CUDA library that enables 8-bit and 4-bit quantization for large language models. Its 8-bit mode keeps statistical outlier features in 16-bit while compressing the rest, and its 4-bit mode powers QLoRA by loading base weights in the NF4 format while LoRA adapters train in 16-bit. It integrates with HuggingFace Transformers via simple load_in_8bit and load_in_4bit flags, making it the go-to tool for memory-efficient fine-tuning.", "fundamentals": "8-bit: per-vector absmax scaling. Outlier decomposition: X = X_normal + X_outlier, compute separately in int8 and fp16, sum results. 4-bit: per-group (g=64 default) with nf4 or fp4 dtype. Double quant: group the fp32 absmax values into blocks of 256, quantize to fp8 with their own absmax.", "seen_in": [ "model-name", "repo-name" ], "related": [ "adapters", "fp16", "fp32", "fp8", "int8", "lora", "nf4", "qlora", "quantization" ], "sources": [ "Dettmers et al., 'LLM.int8(),' arXiv:2208.07339", "Dettmers et al., 'QLoRA,' arXiv:2305.14314" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale", "authors": "Dettmers et al.", "venue": "NeurIPS 2022", "arxiv": "2208.07339" }, { "title": "QLoRA: Efficient Finetuning of Quantized Language Models", "authors": "Dettmers et al.", "venue": "NeurIPS 2023", "arxiv": "2305.14314" } ], "resources": [ { "label": "HuggingFace — bitsandbytes quantization", "url": "https://huggingface.co/docs/transformers/en/quantization/bitsandbytes" } ] }, "calibration-data": { "id": "calibration-data", "name": "Calibration Data", "expansion": "Quantization Calibration Dataset", "category": "quantization-basics", "oneliner": "A small dataset (~128 samples) run through the model to collect activation statistics needed by quantization methods to set scale factors.", "explanation": "Calibration data is a small representative dataset used to measure activation ranges inside a model so quantization methods can set accurate scaling parameters. The standard practice is 128 samples of 2048 tokens, typically from the C4 or WikiText-2 datasets. Methods like GPTQ and AWQ require calibration, while simpler round-to-nearest approaches do not. Using data matched to the deployment domain yields slightly better quantized model quality.", "fundamentals": "For layer Y=XW+b: calibration records per-layer activation stats. Approaches: min/max (simple but outlier-sensitive), percentile (clips outliers), MSE-optimal (minimizes reconstruction error), entropy/KL-based (TensorRT). For GPTQ: calibration computes Hessian H = 2XX^T, used to solve layer-wise optimization min_Q $\\|\\|WX - QX\\|\\|^2$.", "seen_in": [ "documentation", "quantization-scripts" ], "related": [ "awq", "gptq", "imatrix", "quantization" ], "sources": [ "Frantar et al., 'GPTQ,' arXiv:2210.17323", "Nagel et al., arXiv:2106.08295" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "GPTQ paper (established 128-sample C4 calibration standard)", "url": "https://arxiv.org/abs/2210.17323" }, { "label": "White Paper on Neural Network Quantization", "url": "https://arxiv.org/abs/2106.08295" } ] }, "causal-lm": { "id": "causal-lm", "name": "Causal Language Modeling", "expansion": "Causal (Autoregressive) Language Modeling", "category": "training-pipeline", "oneliner": "The training objective where a model predicts the next token using only the tokens to its left — the foundation of GPT-style generation.", "explanation": "Causal language modeling is the training objective where a model predicts the next token using only the tokens to its left, never peeking ahead. A causal attention mask ensures each position can only attend to preceding ones. This left-to-right constraint is the pre-training objective behind GPT, Llama, and Mistral, and it enables autoregressive text generation at inference time.", "fundamentals": "The causal LM objective maximises $\\mathcal{L} = \\sum_{t=1}^{T} \\log P(x_t \\mid x_{>$d^2$, significant reduction. Tiling: for each Q block, iterate K/V blocks. Maintain running max m and sum l for online softmax. Rescale output O as new blocks processed. Memory: $O(N)$ — only Q,K,V,O in HBM, no N$\\times$N matrix. Backward pass: recomputes attention from Q,K,V (gradient checkpointing). Versions: FA1 ~120 TFLOPS A100, FA2 ~230 TFLOPS A100, FA3 ~740 TFLOPS H100.", "seen_in": [ "code", "model-config", "serving-config" ], "related": [ "inference", "kv-cache", "paged-attention", "sdpa" ], "sources": [ "Dao et al., 'FlashAttention,' NeurIPS 2022, arXiv:2205.14135", "Dao, 'FlashAttention-2,' 2023, arXiv:2307.08691", "Shah & Dao et al., 'FlashAttention-3,' 2024, arXiv:2407.08691" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness", "authors": "Dao et al.", "venue": "NeurIPS 2022", "arxiv": "2205.14135" }, { "title": "FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning", "authors": "Dao", "venue": "2023", "arxiv": "2307.08691" } ], "resources": [ { "label": "Lilian Weng — LLM Inference Optimization", "url": "https://lilianweng.github.io/posts/2023-01-10-inference-optimization/" } ] }, "format-coreml": { "id": "format-coreml", "name": "CoreML", "expansion": "Apple's ML format for on-device inference", "category": "formats", "oneliner": ".mlmodel/.mlpackage files. Runs on Apple Neural Engine, GPU, CPU transparently. For iOS/macOS apps. Converter: coremltools.", "seen_in": [ "filename" ], "related": [ "inference", "org-mlx-community" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "Apple Core ML docs", "url": "https://developer.apple.com/documentation/coreml" }, { "label": "coremltools", "url": "https://github.com/apple/coremltools" } ], "explanation": "Core ML is Apple's on-device machine learning framework, using .mlmodel and .mlpackage file formats. It automatically dispatches inference across the Apple Neural Engine, GPU, and CPU for optimal performance on iPhones, iPads, and Macs. Developers integrate models directly into iOS and macOS apps with a few lines of Swift, and Apple's tools handle hardware-specific optimization transparently." }, "format-ct2": { "id": "format-ct2", "name": "CTranslate2 (ct2)", "expansion": "Efficient inference engine/format by OpenNMT", "category": "formats", "oneliner": "Fast CPU inference, INT8/INT16/FP16. Originally built for translation (encoder-decoder). Less common for large decoder-only LLMs. Reduced activity since late 2024.", "seen_in": [ "filename" ], "related": [ "fp16", "inference", "inference-engine", "int8", "quantization", "tool-llamacpp", "tool-vllm" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "CTranslate2 GitHub", "url": "https://github.com/OpenNMT/CTranslate2" } ], "explanation": "CTranslate2 is a lightweight inference engine optimized for fast CPU execution with INT8, INT16, and FP16 quantization. Originally developed by OpenNMT for translation models like MarianMT and NLLB, it also supports encoder-decoder architectures such as Whisper. However, it has seen reduced development activity since late 2024 and is less commonly used for large decoder-only LLMs compared to llama.cpp or vLLM." }, "format-onnx": { "id": "format-onnx", "name": "ONNX", "expansion": "Open Neural Network Exchange", "category": "formats", "oneliner": "Open format for ML computation graphs. Train in PyTorch, export to ONNX, run anywhere via ONNX Runtime (CPU, CUDA, DirectML, TensorRT). Key for edge/Windows.", "seen_in": [ "filename", "model-name" ], "related": [ "inference", "quantization" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "ONNX GitHub", "url": "https://github.com/onnx/onnx" }, { "label": "ONNX Runtime", "url": "https://github.com/microsoft/onnxruntime" } ], "explanation": "ONNX (Open Neural Network Exchange) is an open standard for representing machine learning computation graphs. You train a model in any framework like PyTorch or TensorFlow, export it to ONNX format, and run inference anywhere using ONNX Runtime. It supports operator-level optimizations and quantization, making it a practical choice for deploying models across CPUs, GPUs, and edge devices without framework lock-in." }, "format-openvino": { "id": "format-openvino", "name": "OpenVINO", "expansion": "Intel's inference optimization toolkit", "category": "formats", "oneliner": "Converts to OpenVINO IR format, applies INT8/INT4 quant via NNCF, hardware-specific kernels. Significantly faster than generic ONNX on Intel CPUs (AMX).", "seen_in": [ "filename" ], "related": [ "format-onnx", "inference", "int4", "int8", "quantization" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "OpenVINO GitHub", "url": "https://github.com/openvinotoolkit/openvino" } ], "explanation": "OpenVINO is Intel's toolkit for optimizing and deploying inference workloads on Intel hardware. It converts models from PyTorch or ONNX into its own Intermediate Representation format, then applies INT8 and INT4 quantization along with graph-level optimizations. The result is significantly faster inference on Intel CPUs and integrated GPUs, making it the go-to choice for deploying AI on Intel-based servers and edge devices." }, "fp16": { "id": "fp16", "name": "fp16 / float16", "expansion": "16-bit IEEE 754 Half-Precision Floating Point", "category": "precision-formats", "oneliner": "A 16-bit float that halves memory vs fp32 but has limited dynamic range — fast for inference, needs loss scaling for training.", "explanation": "FP16 is a 16-bit floating-point format that halves memory compared to 32-bit, so a 7-billion parameter model fits in about 14 GB. It enables tensor core acceleration on NVIDIA GPUs for significant throughput gains. The main limitation is a small maximum value of 65,504, meaning training gradients can overflow and require loss scaling. For inference this is rarely a problem. FP16 has largely been replaced by BFloat16 for training.", "fundamentals": "Bit layout: 1 sign | 5 exponent (bias 15) | 10 mantissa. Value = (-1)^sign $\\times$ 2^(exp-15) $\\times$ 1.mantissa. Range: $\\pm$65,504. Precision: ~3.3 decimal digits. Compared to fp32: range drops from ~$10^{38}$ to ~6.5$\\times 10^{4}$ (34 orders of magnitude less), precision from ~7.2 to ~3.3 digits.", "seen_in": [ "model-name", "filename" ], "related": [ "bf16", "fp32", "inference" ], "sources": [ "IEEE 754-2008 Section 3.6", "Micikevicius et al., 'Mixed Precision Training,' ICLR 2018, arXiv:1710.03740" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "Mixed Precision Training", "authors": "Micikevicius et al.", "venue": "ICLR 2018", "arxiv": "1710.03740" } ] }, "fp32": { "id": "fp32", "name": "fp32 / float32", "expansion": "32-bit IEEE 754 Single-Precision Floating Point", "category": "precision-formats", "oneliner": "The standard 32-bit floating-point format — baseline 'full precision' for neural networks.", "explanation": "FP32 is the standard 32-bit floating-point format where each parameter occupies 4 bytes, so a 7-billion parameter model requires about 28 GB. It provides excellent precision and enormous numerical range but is now considered overkill for model weights. It is still used for optimizer states and master weight copies in mixed-precision training. When people say a model is quantized, they mean converted from FP32 or BFloat16 to a smaller format.", "fundamentals": "Bit layout: 1 sign | 8 exponent (bias 127) | 23 mantissa. Value = (-1)^sign $\\times$ 2^(exp-127) $\\times$ 1.mantissa. The 23-bit mantissa + implicit leading 1 = 24 bits of significand $\\approx$ 7.2 decimal digits. Special values: $\\pm$inf (exp all 1s, mantissa 0), NaN (exp all 1s, mantissa ≠0), subnormals (exp all 0s).", "seen_in": [ "model-weights", "optimizer-states" ], "related": [ "fp16", "bf16", "quantization" ], "sources": [ "IEEE 754-2019" ], "confidence": "high", "verified_date": "2026-04-10", "resources": [ { "label": "IEEE 754-2019 Standard", "url": "https://ieeexplore.ieee.org/document/8766229" } ] }, "fp8": { "id": "fp8", "name": "fp8 (E4M3 / E5M2)", "expansion": "8-bit Floating Point — two variants for forward and backward passes", "category": "precision-formats", "oneliner": "Two 8-bit float formats: E4M3 for weights/activations (more precision) and E5M2 for gradients (more range). 2$\\times$ throughput vs fp16.", "explanation": "FP8 is an 8-bit floating-point format available in two variants optimized for different parts of the training loop. The E4M3 variant provides more precision for forward-pass weights and activations, while E5M2 provides more dynamic range for backward-pass gradients. FP8 runs natively on NVIDIA H100 tensor cores at double the throughput of FP16, with accuracy loss typically under 0.1 percent.", "fundamentals": "E4M3: 1 sign | 4 exp (bias 7) | 3 mantissa. No infinity — all exp=1111 patterns are finite (deliberate departure from IEEE 754). Only 16 values per sign per binade. E5M2: 1 sign | 5 exp (bias 15) | 2 mantissa. Follows IEEE 754 conventions (has inf/NaN). Per-tensor scaling factors are essential — without them, clipping destroys quality.", "seen_in": [ "model-name", "serving-config" ], "related": [ "bf16", "fp16", "w8a8" ], "sources": [ "Micikevicius et al., 'FP8 Formats for Deep Learning,' arXiv:2209.05433" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "FP8 Formats for Deep Learning", "authors": "Micikevicius et al.", "venue": "2022", "arxiv": "2209.05433" } ] }, "full-ft": { "id": "full-ft", "name": "Full Fine-Tuning", "expansion": "Full Fine-Tuning (all parameters updated)", "category": "fine-tuning-methods", "oneliner": "Update every parameter — ~12N bytes memory, maximum expressiveness, risks catastrophic forgetting, produces full model copies.", "explanation": "Full fine-tuning is the process of updating every parameter in a model during training, giving maximum flexibility to learn new tasks or domains. It requires roughly twelve times the model size in memory to hold weights, gradients, optimizer states, and master copies. A 7B parameter model needs around 84 GB, so distributed training with DeepSpeed ZeRO or FSDP is essential. All major instruct and chat models from leading labs use full fine-tuning.", "fundamentals": "Memory: 7B→84GB, 13B→156GB, 70B→840GB. ZeRO-1: shard optimizer. ZeRO-2: +gradients. ZeRO-3: +parameters. lr 1e-5 to 5e-5, 1-3 epochs.", "seen_in": [ "model-name" ], "related": [ "lora", "sft", "zero" ], "foundational_papers": [ { "title": "Training language models to follow instructions with human feedback (InstructGPT)", "authors": "Ouyang et al.", "venue": "NeurIPS 2022", "arxiv": "2203.02155" }, { "title": "ZeRO: Memory Optimizations Toward Training Trillion Parameter Models", "authors": "Rajbhandari et al.", "venue": "SC 2020", "arxiv": "1910.02054" } ], "sources": [ "Ouyang et al., arXiv:2203.02155" ], "confidence": "high", "verified_date": "2026-04-10" }, "function-calling": { "id": "function-calling", "name": "Function Calling", "expansion": "LLM Function Calling / Tool Use", "category": "agents-and-tools", "oneliner": "The mechanism by which an LLM emits structured JSON to invoke external tools instead of generating free-text. Foundation of all agentic behaviour.", "explanation": "Function calling is the capability that lets a language model request execution of external functions by outputting structured arguments rather than plain text. The developer registers a set of tool schemas (name, description, parameter JSON Schema) in the prompt or API call. When the model decides a tool is needed, it emits a tool-call object with the function name and arguments. The harness executes the function, returns the result, and the model continues reasoning.", "fundamentals": "Flow: user prompt + tool schemas -> model reasons -> model emits tool_call(name, args) -> harness validates and executes -> result appended as tool_result -> model generates final answer or chains another call. Parallel tool calling lets the model emit multiple calls in one turn. Forced tool use constrains the model to always call a specific function. Structured output mode (JSON Schema) is closely related but produces data rather than invoking actions. Fine-tuning on function-calling datasets improves reliability; constrained decoding guarantees valid JSON.", "related": [ "agentic-ai", "mcp", "inference-engine" ], "seen_in": [ "documentation", "code" ], "foundational_papers": [ { "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "authors": "Schick et al.", "venue": "NeurIPS 2023", "arxiv": "2302.04761" }, { "title": "Gorilla: Large Language Model Connected with Massive APIs", "authors": "Patil et al.", "venue": "arXiv 2023", "arxiv": "2305.15334" } ], "resources": [ { "label": "OpenAI function calling guide", "url": "https://platform.openai.com/docs/guides/function-calling" }, { "label": "Anthropic tool use docs", "url": "https://docs.anthropic.com/en/docs/build-with-claude/tool-use" } ], "confidence": "high", "verified_date": "2026-04-11" }, "geglu": { "id": "geglu", "name": "GeGLU", "expansion": "GELU-activated Gated Linear Unit", "category": "layer-types", "oneliner": "GLU variant using GELU activation instead of Swish — performs nearly identically to SwiGLU, used in T5 v1.1 and Flan-T5.", "explanation": "GeGLU is a gated activation function that uses the GELU nonlinearity as its gating mechanism inside the feed-forward network. It was introduced as an alternative to ReLU and performs nearly identically to SwiGLU in quality benchmarks. The T5 and Flan-T5 model families adopted GeGLU, while the Llama ecosystem standardized on SwiGLU instead. It has become less common in decoder-only LLMs since 2023.", "fundamentals": "GeGLU(x) = (GELU(xW_gate) ⊙ xW_up)W_down. GELU(z) $\\approx$ z·0.5·(1 + tanh($\\sqrt{2/π}$·(z + 0.044715$z^3$))). Same parameter count as SwiGLU: 3 $\\times$ d_model $\\times$ d_ff. Swish vs GELU: $\\Phi$(x) has slightly steeper transition than $\\sigma$(x), negligible practical difference.", "seen_in": [ "model-config", "code" ], "related": [ "ffn", "swiglu" ], "sources": [ "Shazeer, 'GLU Variants Improve Transformer,' 2020, arXiv:2002.05202", "Hendrycks & Gimpel, 'GELUs,' 2016, arXiv:1606.08415" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "GLU Variants Improve Transformer", "authors": "Shazeer", "venue": "2020", "arxiv": "2002.05202" }, { "title": "Gaussian Error Linear Units (GELUs)", "authors": "Hendrycks & Gimpel", "venue": "2016", "arxiv": "1606.08415" } ] }, "ggml": { "id": "ggml", "name": "GGML", "expansion": "Georgi Gerganov Machine Learning (library and legacy format)", "category": "formats", "oneliner": "Legacy format — predecessor to GGUF. No longer used for new models but you may encounter old repos with .ggml files.", "explanation": "GGML is an obsolete model file format that was originally used by llama.cpp for local inference. It stored tensor data but lacked proper metadata support, with no fields for tokenizer configuration, model architecture details, or format versioning. The GGML C library still powers tensor math in llama.cpp, but the file format was replaced by GGUF in August 2023.", "fundamentals": "Simple binary layout: tensor data with minimal headers. No standardized metadata — required external configuration files. No versioning — breaking changes required new tooling. GGUF solved all of these by adding a proper header with KV metadata.", "seen_in": [ "legacy-repos", "old-filenames" ], "related": [ "gguf", "inference", "tool-llamacpp" ], "sources": [ "github.com/ggerganov/ggml" ], "confidence": "high", "verified_date": "2026-04-10" }, "gguf": { "id": "gguf", "name": "GGUF", "expansion": "GGML Universal File", "category": "formats", "oneliner": "The standard single-file format for llama.cpp/Ollama/LM Studio — contains model weights, metadata, tokenizer, and quantization info in one file.", "explanation": "GGUF is a self-contained model file format designed for local inference with llama.cpp and compatible tools. Everything needed to load and run a model lives in a single file, including weights, tokenizer data, and architecture metadata stored as key-value pairs. It supports all common quantization types from Q2_K through Q8_0, plus IQ and UD variants. GGUF is the standard format for split CPU and GPU inference on consumer hardware.", "fundamentals": "File structure: 1) Magic number ('GGUF'). 2) Version. 3) Tensor count. 4) Metadata KV pairs (architecture, tokenizer, quant type, etc.). 5) Tensor descriptors (name, shape, type, offset). 6) Tensor data (aligned, mmap-friendly). Quantization type is per-tensor — different tensors can use different types. Supports memory mapping for efficient loading.", "seen_in": [ "repo-name", "filename" ], "related": [ "ggml", "inference", "iq-quants", "k-quants", "quantization", "tool-llamacpp", "tool-lmstudio", "tool-ollama" ], "sources": [ "github.com/ggml-org/ggml/blob/master/docs/gguf.md", "gguf-py README" ], "confidence": "high", "verified_date": "2026-04-10" }, "gpqa": { "id": "gpqa", "name": "GPQA", "expansion": "GPQA — Graduate-Level Google-Proof Q&A", "category": "scaling-patterns", "oneliner": "A 448-question science reasoning benchmark where non-expert humans score only 34% — designed to measure frontier model capabilities that MMLU can no longer distinguish.", "explanation": "GPQA is a benchmark of 448 extremely difficult science questions in biology, physics, and chemistry, written by domain experts and verified to be unsearchable. Non-expert humans with internet access score only 34 percent, while expert humans score around 65 percent. GPQA has replaced MMLU as the discriminating benchmark for frontier models because MMLU is now saturated above 88 percent. Models are evaluated on the GPQA-Diamond subset of 198 hardest questions.", "related": [ "mmlu", "evals", "reasoning-models" ], "seen_in": [ "model-cards", "documentation" ], "foundational_papers": [ { "title": "GPQA: A Graduate-Level Google-Proof Q&A Benchmark", "authors": "Rein et al.", "venue": "2023", "arxiv": "2311.12022" } ], "resources": [ { "label": "GPQA paper", "url": "https://arxiv.org/abs/2311.12022" } ], "confidence": "high", "verified_date": "2026-04-11" }, "gptq": { "id": "gptq", "name": "GPTQ", "expansion": "GPT-Quantization (named after target: Generative Pre-trained Transformers)", "category": "quantization-methods", "oneliner": "Post-training weight quantization using second-order (Hessian) information to compress weights to 3-4 bits with minimal accuracy loss.", "explanation": "GPTQ is a post-training quantization method that compresses model weights layer by layer using mathematical error correction. When one weight is rounded to a lower precision, GPTQ adjusts the remaining weights in that layer to compensate, guided by Hessian information about which errors matter most. Processing columns in blocks of 128 makes it scale to models with hundreds of billions of parameters in just a few GPU-hours.", "fundamentals": "Objective: $\\min \\|WX - Q(W)X\\|^2$. OBS update: when $w_q$ is quantized, distribute error via $\\delta w = -\\delta \\cdot (H^{-1}_{:,q} / H^{-1}_{q,q})$. GPTQ processes columns left-to-right with lazy batch updates per 128-column block. Complexity: $O(d_{\\text{row}} \\times d_{\\text{col}}^2)$.", "seen_in": [ "model-name", "filename" ], "related": [ "awq", "marlin", "ptq", "quantization" ], "sources": [ "Frantar et al., 'GPTQ,' ICLR 2023, arXiv:2210.17323" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "GPTQ: Accurate Post-Training Quantization for Generative Pre-Trained Transformers", "authors": "Frantar et al.", "venue": "ICLR 2023", "arxiv": "2210.17323" } ], "resources": [ { "label": "HuggingFace — Selecting a Quantization Method", "url": "https://huggingface.co/docs/transformers/quantization/selecting" } ] }, "gpu-memory": { "id": "gpu-memory", "name": "GPU Memory", "expansion": "GPU Memory (VRAM) for LLMs", "category": "scaling-patterns", "oneliner": "The GPU video memory (VRAM) available for loading model weights, KV cache, and activations — the primary hardware constraint for LLM deployment.", "explanation": "GPU memory (VRAM) is the single most important hardware constraint for running LLMs. Model weights, the KV cache, and activations must all fit in VRAM. Consumer GPUs like the RTX 4090 have 24 GB, datacenter GPUs like A100 have 40-80 GB, and H100s have 80 GB of fast HBM3 memory. A 70B model in fp16 needs 140 GB — more than any single GPU — requiring either quantization (4-bit reduces it to 35 GB) or multi-GPU distribution.", "related": [ "model-size-memory", "quantization", "int4", "tp-pp", "inference" ], "seen_in": [ "documentation" ], "foundational_papers": [], "resources": [], "confidence": "high", "verified_date": "2026-04-11" }, "gqa": { "id": "gqa", "name": "GQA", "expansion": "Grouped Query Attention", "category": "attention-variants", "oneliner": "Query heads grouped into G groups, each sharing one K/V head — the dominant middle-ground between MHA quality and MQA efficiency.", "explanation": "Grouped Query Attention is a middle ground between standard multi-head attention and multi-query attention that groups query heads into clusters, with each group sharing one key and one value head. This reduces KV cache size without the quality loss of using a single shared head. Existing multi-head models can be converted to GQA with just 5-10 percent of the original training compute. It is the dominant attention variant in modern LLMs including Llama 2 70B, Llama 3, and Mistral.", "fundamentals": "For Mistral 7B: hidden_dim=4096, n_query_heads=32, n_kv_heads=8, head_dim=128, G=4. W_Q [4096, 4096], W_K [4096, 1024], W_V [4096, 1024]. Query heads 0-3 attend to KV head 0, heads 4-7 to KV head 1, etc. Implementation: K/V repeated via repeat_interleave or grouped computation. KV cache per token per layer: 2 $\\times$ 8 $\\times$ 128 $\\times$ 2 = 4 KB (4$\\times$ reduction vs MHA, 8$\\times$ larger than MQA). General formula: cache = 2 $\\times$ n_kv_heads $\\times$ head_dim $\\times$ seq_len $\\times$ n_layers $\\times$ bytes_per_param.", "seen_in": [ "model-config" ], "related": [ "kv-cache", "mha", "mla", "mqa", "token" ], "sources": [ "Ainslie et al., 'GQA: Training Generalized Multi-Query Transformer Models,' EMNLP 2023, arXiv:2305.13245" ], "confidence": "high", "verified_date": "2026-04-10", "foundational_papers": [ { "title": "GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints", "authors": "Ainslie et al.", "venue": "EMNLP 2023", "arxiv": "2305.13245" } ], "resources": [ { "label": "Sebastian Raschka — Visual Attention Variants", "url": "https://magazine.sebastianraschka.com/p/visual-attention-variants" }, { "label": "Lilian Weng — The Transformer Family v2", "url": "https://lilianweng.github.io/posts/2023-01-27-the-transformer-family-v2/" } ] }, "gradient-checkpointing": { "id": "gradient-checkpointing", "name": "Gradient Checkpointing", "expansion": "Gradient Checkpointing (Activation Recomputation)", "category": "training-pipeline", "oneliner": "A memory optimization that trades compute for memory by recomputing activations during the backward pass instead of storing them all.", "explanation": "Gradient checkpointing saves GPU memory during training by discarding intermediate activations in the forward pass and recomputing them during the backward pass. Without it, a transformer stores activations from every layer, consuming tens of gigabytes. Checkpointing roughly halves activation memory at the cost of about 33 percent extra compute. It is essential for training large models on limited hardware and is enabled by a single flag in most frameworks.", "fundamentals": "Standard: store all $L$ layers' activations, memory $O(L \\cdot B \\cdot S \\cdot D)$ where $B$ = batch, $S$ = seq_len, $D$ = hidden_dim. With checkpointing every $\\sqrt{L}$ layers: memory $O(\\sqrt{L} \\cdot BSD)$, compute increases by ~33% (one extra forward pass). Selective checkpointing: only checkpoint attention layers (expensive to store) while keeping cheap layers (norms, residuals). Flash Attention's recomputation of attention in the backward pass is a form of selective checkpointing.", "related": [ "flash-attention", "full-ft", "zero", "mixed-precision-training" ], "seen_in": [ "training-config", "code" ], "foundational_papers": [ { "title": "Training Deep Nets with Sublinear Memory Cost", "authors": "Chen et al.", "venue": "2016", "arxiv": "1604.06174" } ], "resources": [ { "label": "Lilian Weng — How to Train Really Large Models", "url": "https://lilianweng.github.io/posts/2021-09-25-train-large/" } ], "confidence": "high", "verified_date": "2026-04-11" }, "greedy-decoding": { "id": "greedy-decoding", "name": "Greedy Decoding", "expansion": "Greedy Decoding (Argmax Sampling)", "category": "sampling-decoding", "oneliner": "The simplest decoding strategy — always pick the single highest-probability token at each step. Fast and deterministic but can produce repetitive or suboptimal text.", "explanation": "Greedy decoding selects the token with the highest probability at each generation step, equivalent to setting temperature to 0. It produces deterministic output — the same input always gives the same output. While fast and predictable, greedy decoding often misses better sequences that would have been found by exploring lower-probability tokens early on. It tends to produce repetitive text and can get stuck in loops.", "related": [ "temperature", "top-p", "top-k" ], "seen_in": [ "code", "documentation" ], "foundational_papers": [], "resources": [], "confidence": "high", "verified_date": "2026-04-11" }, "grpo": { "id": "grpo", "name": "GRPO", "expansion": "Group Relative Policy Optimization", "category": "training-pipeline", "oneliner": "PPO variant that drops the critic model — estimates advantages by normalizing rewards across a group of sampled outputs for the same prompt. Powers DeepSeek R1.", "explanation": "Group Relative Policy Optimization is an RL algorithm that simplifies PPO by removing the critic model entirely. For each prompt, GRPO samples a group of outputs, scores them, then normalizes rewards within the group to estimate advantages. This replaces the learned value function, cutting memory by 40 to 60 percent. Introduced in DeepSeekMath, GRPO became the training algorithm for DeepSeek R1, where it trained reasoning from scratch using only correctness rewards.", "fundamentals": "Sample group $\\{o_1, \\ldots, o_G\\}$ from $\\pi_{\\theta_{\\text{old}}}(\\cdot|q)$. Compute rewards $r_i$. Normalize: $\\hat{A}_i = \\frac{r_i - \\mathrm{mean}(\\mathbf{r})}{\\mathrm{std}(\\mathbf{r})}$. Clipped objective per token $t$: $\\mathcal{L} = -\\frac{1}{G}\\sum_i \\frac{1}{|o_i|}\\sum_t \\min\\!\\bigl(\\rho_t \\hat{A}_i,\\; \\mathrm{clip}(\\rho_t, 1\\pm\\varepsilon)\\hat{A}_i\\bigr) + \\beta\\,\\mathrm{KL}(\\pi_\\theta \\| \\pi_{\\mathrm{ref}})$ where $\\rho_t = \\pi_\\theta(o_t|q,o_{