{ "schemaVersion": "0.1.0", "researchAccessDate": "2026-09-17", "scope": "Educational reference, overlapping hardware families; economics deferred", "architectures": [ { "id": "cpu", "name": "General-purpose CPU with vector or matrix extensions", "category": "Instruction-driven digital compute", "maturity": "Established commercial hardware; AI acceleration varies by processor and software.", "training": "Runs complete training pipelines; useful for preprocessing, classical ML and smaller neural models. Large dense-model training usually benefits from accelerators.", "inference": "Runs broad model types, including quantized neural networks; often also hosts tokenization, request scheduling and application logic.", "components": [ "CPU cores", "Vector or matrix execution units", "Registers", "L1/L2/L3 caches", "Memory controller", "DDR DRAM", "PCIe root complex", "Network interface", "Storage" ], "interactions": [ "Storage supplies examples or model weights to host DRAM.", "Cores fetch instructions; caches feed scalar, vector and matrix units, which write results back through the memory hierarchy.", "During training, forward computation produces a loss, backward computation produces gradients, and the optimizer updates weights.", "A host CPU can instead prepare work and submit commands to attached accelerators over PCIe.", "Multiple sockets or servers exchange tensors through socket links or network interfaces." ], "memoryModel": "Cache hierarchy in front of system DRAM; on multisocket servers, memory locality and NUMA placement matter.", "interconnect": "On-chip fabric, DRAM channels, socket links, PCIe and Ethernet or other server networking.", "bottlenecks": [ "DRAM bandwidth and cache misses", "Vector or matrix utilization", "NUMA traffic", "Thread scheduling and preprocessing throughput" ], "examples": [ "Intel Xeon with AMX" ], "visualNotes": "Open a CPU package into cores, caches and memory channels. Animate a cache hit and a slower DRAM access; add an optional accelerator offload path.", "sourceIds": [ "hw-intel-amx", "hw-intel-ai-tuning" ], "history": { "id": "cpu", "pioneers": [ { "name": "University of Manchester", "kind": "organisation", "contribution": "Stored-program computing; Williams, Kilburn and Tootill team", "sourceIds": [ "hist-manchester" ] }, { "name": "Freddie Williams, Tom Kilburn and Geoff Tootill", "kind": "individuals", "contribution": "Manchester Baby design and construction", "sourceIds": [ "hist-manchester" ] }, { "name": "Seymour Cray and Cray Research", "kind": "team", "contribution": "Commercial vector supercomputing", "sourceIds": [ "hist-cray" ] } ], "milestones": [ { "date": "1948-06-21", "precision": "day", "type": "Demonstration", "title": "Manchester Baby runs a stored program", "description": "An early operational foundation for general-purpose stored-program computers.", "sourceIds": [ "hist-manchester" ] }, { "date": "1976", "precision": "year", "type": "Deployment", "title": "Cray-1 at Los Alamos", "description": "A major vector-computing milestone, distinct from the later addition of vector and matrix extensions to commodity CPUs.", "sourceIds": [ "hist-cray" ] } ], "note": "Selected milestones in a broad lineage, not a single invention date for CPUs, vector processing or modern matrix instructions." } }, { "id": "discrete-gpu", "name": "Discrete GPU accelerator", "category": "Instruction-driven digital compute", "maturity": "Established commercial hardware for training and inference, from workstations to large clusters.", "training": "Parallel kernels execute forward, backward and optimizer operations; collectives synchronize distributed training.", "inference": "Supports diverse models and both prompt processing and token generation; batching, precision and memory layout affect performance.", "components": [ "Host CPU", "GPU compute units", "Tensor or matrix cores", "Registers and shared SRAM", "L2 cache", "HBM or GDDR", "DMA engines", "PCIe interface", "Accelerator links", "NIC and switches" ], "interactions": [ "The CPU launches kernels and arranges transfers; DMA moves inputs and weights into device memory.", "Compute units stage matrix tiles from device memory (HBM or GDDR, depending on product) through on-chip memory and execute many parallel operations.", "Training produces gradients that peer GPUs combine through collective communication.", "Autoregressive transformer inference with attention caching repeatedly reads weights and attention KV state and appends new KV entries.", "Scale-up links connect nearby accelerators; network interfaces and switches connect servers." ], "memoryModel": "Device memory is usually distinct from host DRAM. Registers, scratchpads and caches reduce repeated device-memory reads; coherent host-device designs are a variant.", "interconnect": "PCIe to host; vendor links such as NVLink or Infinity Fabric between accelerators; Ethernet/RDMA or InfiniBand across nodes.", "bottlenecks": [ "Device-memory bandwidth and capacity", "Matrix utilization and kernel launch overhead", "Collective latency and network congestion", "Power and heat removal" ], "examples": [ "NVIDIA Blackwell B200", "AMD Instinct MI300X" ], "visualNotes": "Explode a board into GPU package, surrounding memory and links; zoom into tiled compute units. Show weight reads, KV growth and a gradient all-reduce as separate modes.", "sourceIds": [ "hw-nvidia-blackwell", "hw-nvidia-system", "hw-amd-mi300" ], "history": { "id": "discrete-gpu", "pioneers": [ { "name": "NVIDIA", "kind": "organisation", "contribution": "Modern GPU product milestone and CUDA ecosystem", "sourceIds": [ "hist-nvidia" ] } ], "milestones": [ { "date": "1999", "precision": "year", "type": "Product introduction", "title": "GeForce-era GPU milestone", "description": "NVIDIA dates its modern GPU milestone to 1999; this is its product lineage, not the beginning of all graphics acceleration.", "sourceIds": [ "hist-nvidia" ] }, { "date": "2006", "precision": "year", "type": "Platform announcement", "title": "CUDA architecture unveiled", "description": "Programmable GPU parallel computing becomes a platform for scientific and general-purpose workloads.", "sourceIds": [ "hist-nvidia" ] } ], "note": "Graphics processing and general-purpose GPU research have many contributors. NVIDIA is a selected commercial pioneer, not the sole inventor of graphics acceleration." } }, { "id": "integrated-soc", "name": "Integrated CPU, GPU and NPU system-on-chip", "category": "Integration and memory organization", "maturity": "Established commercial design in phones, PCs and embedded devices; data-center APUs are a related integration variant.", "training": "CPU/GPU paths can train or fine-tune models that fit available resources. NPU training support is product-specific and must not be assumed.", "inference": "A runtime can assign supported operations to CPU, GPU or NPU, balancing latency, energy and operator availability.", "components": [ "CPU cores", "Integrated GPU", "NPU or neural engine", "Shared memory controllers", "Unified DRAM", "On-chip fabric", "Caches", "Media and sensor interfaces", "Power management" ], "interactions": [ "Inputs arrive from storage, networking or sensors and enter shared memory.", "A framework schedules compatible kernels on a selected compute engine.", "CPU and GPU can use the same underlying allocation on unified-memory systems, avoiding some explicit copies.", "Engines still synchronize dependencies and compete for memory bandwidth and power.", "Unsupported operations may fall back to another engine; exact partitioning depends on the runtime." ], "memoryModel": "Shared physical DRAM with caches and synchronization rules; unified memory does not mean unlimited bandwidth or identical access costs.", "interconnect": "On-chip or package fabric links compute engines and memory controllers; external I/O remains available.", "bottlenecks": [ "Shared memory bandwidth", "Memory capacity", "Thermal and battery limits", "Operator support and cross-engine synchronization" ], "examples": [ "Apple M4 with CPU, GPU and Neural Engine", "AMD MI300A CPU/GPU APU; not an NPU example" ], "visualNotes": "Lift a package lid to reveal several engines around one memory pool. Do not draw shared memory as separate CPU and GPU copies or imply every operation uses the NPU.", "sourceIds": [ "hw-apple-m4", "hw-apple-mlx", "hw-amd-mi300a" ], "history": { "id": "integrated-soc", "pioneers": [ { "name": "Apple silicon engineering team", "kind": "organisation", "contribution": "A representative CPU + GPU + neural-engine integration milestone", "sourceIds": [ "hist-apple-a11" ] } ], "milestones": [ { "date": "2017-09-12", "precision": "day", "type": "Product announcement", "title": "A11 Bionic announced with a Neural Engine", "description": "Apple presents CPU, GPU and dedicated neural processing in its A11 Bionic design.", "sourceIds": [ "hist-apple-a11" ] } ], "note": "This date marks one influential integrated AI SoC product. It does not credit Apple with inventing the SoC, heterogeneous computing or every CPU/GPU/NPU integration." } }, { "id": "tensor-asic", "name": "Tensor ASIC with systolic matrix arrays", "category": "Specialized digital compute", "maturity": "Established commercial cloud hardware; capabilities and topology differ across generations.", "training": "Supported generations execute large distributed training jobs using matrix, vector and scalar engines plus collective communication.", "inference": "Also serves neural networks; a product may be tuned for training, inference or both, so the ASIC label alone does not decide the workload.", "components": [ "Host CPU", "Matrix multiplication arrays", "Vector engines", "Scalar engines", "On-chip SRAM", "HBM", "DMA engines", "Collective communication engines", "Inter-chip links" ], "interactions": [ "The compiler tiles an operation graph and assigns work to specialized engines.", "HBM supplies matrix tiles; adjacent multiply-accumulate cells reuse operands as data propagates through an array.", "Vector and scalar engines handle operations such as activations, reductions and control.", "Partial results return to local buffers or HBM; chips exchange tensors over their dedicated fabric.", "Training adds gradients and optimizer updates; inference keeps trained weights and updates request state." ], "memoryModel": "Explicitly managed on-chip buffers backed by HBM; compilers schedule reuse and transfers.", "interconnect": "Google TPU ICI within slices and data-center networking between slices; AWS NeuronLink is a different implementation.", "bottlenecks": [ "Poorly shaped matrix tiles", "HBM bandwidth and capacity", "Unsupported or inefficient operators", "Inter-chip communication" ], "examples": [ "Google Cloud TPU", "AWS Trainium2" ], "visualNotes": "Animate a wave of operands through a grid with accumulating outputs. Offer a chip-to-slice zoom; use a labeled vendor example instead of presenting one topology as universal.", "sourceIds": [ "hw-google-tpu", "hw-aws-trainium2", "hw-aws-glossary" ], "history": { "id": "tensor-asic", "pioneers": [ { "name": "H. T. Kung and Charles E. Leiserson", "kind": "individuals", "contribution": "Systolic-array research at Carnegie Mellon University", "sourceIds": [ "hist-systolic" ] }, { "name": "Google TPU team", "kind": "organisation", "contribution": "Datacenter neural-network ASIC deployment", "sourceIds": [ "hist-tpu" ] } ], "milestones": [ { "date": "1978-04", "precision": "month", "type": "Research report", "title": "Systolic Arrays for (VLSI)", "description": "Kung and Leiserson describe systolic arrays in a Carnegie Mellon report dated April 1978.", "sourceIds": [ "hist-systolic" ] }, { "date": "2015", "precision": "year", "type": "Deployment", "title": "First-generation TPU in Google datacenters", "description": "Google’s 2017 paper reports inference TPU deployment beginning in 2015. The deployment year is not the paper’s publication year.", "sourceIds": [ "hist-tpu" ] } ], "note": "Systolic-array foundations and the commercial TPU are different milestones. Tensor ASICs are a wider family than Google’s implementations." } }, { "id": "reconfigurable-dataflow", "name": "Reconfigurable dataflow accelerator", "category": "Specialized digital compute", "maturity": "Commercial systems; this entry uses SambaNova RDU as a concrete implementation.", "training": "SN40L research describes support for training and inference. Support on a particular service or newer product must be checked separately.", "inference": "Current RDU products emphasize serving large models and moving model data through several memory tiers.", "components": [ "Host CPU", "Programmable compute units", "Programmable SRAM units", "Configurable switch fabric", "Address and communication units", "HBM", "DDR DRAM", "Inter-RDU links", "Graph compiler" ], "interactions": [ "The compiler maps connected operators onto a grid of compute and memory units.", "An operator streams intermediate values toward its consumer while other data transfers overlap with computation.", "Frequently used tiles reside in SRAM; larger state moves through HBM and DDR tiers.", "Inter-RDU links carry graph edges that cross chip boundaries.", "Training maps backward and update computations too; inference schedules weight and request-state movement." ], "memoryModel": "SN40L uses distributed on-chip SRAM, on-package HBM and off-package DDR; tier placement is central to execution.", "interconnect": "Configurable on-chip dataflow routing and a dedicated network between RDUs.", "bottlenecks": [ "Compiler placement and routing", "Available SRAM for fused operators", "Memory-tier transfers", "Graph partitions crossing devices" ], "examples": [ "SambaNova SN40L RDU", "SambaNova SN50 inference RDU" ], "visualNotes": "Morph a model graph into a physical grid. Highlight producer-to-consumer streams and let the viewer expand SRAM, HBM and DDR as separate tiers.", "sourceIds": [ "hw-sambanova-dataflow", "hw-sambanova-sn40l", "hw-sambanova-rdu" ], "history": { "id": "reconfigurable-dataflow", "pioneers": [ { "name": "Jack B. Dennis and David P. Misunas", "kind": "individuals", "contribution": "Foundational data-flow processor work at MIT", "sourceIds": [ "hist-dataflow" ] }, { "name": "SambaNova Systems", "kind": "organisation", "contribution": "Modern reconfigurable dataflow units for AI", "sourceIds": [ "hist-sn40l" ] } ], "milestones": [ { "date": "1974", "precision": "year", "type": "Research publication", "title": "Basic data-flow processor architecture", "description": "ACM’s record dates Dennis and Misunas’s foundational processor paper to 1974; the work is also commonly cited with the 1975 symposium year.", "sourceIds": [ "hist-dataflow" ] }, { "date": "2023-09-19", "precision": "day", "type": "Product announcement", "title": "SambaNova announces SN40L", "description": "A product milestone for the RDU example in this atlas; not the first dataflow processor.", "sourceIds": [ "hist-sn40l" ] } ], "note": "Foundational dataflow execution predates modern reconfigurable AI arrays. SN40L is a representative implementation, not the invention of dataflow." } }, { "id": "wafer-scale", "name": "Wafer-scale many-core accelerator", "category": "Integration and memory organization", "maturity": "Commercial training and inference systems; wafer scale is a packaging/integration lens that overlaps other accelerator categories.", "training": "Cerebras systems use large on-wafer compute and memory arrays; weight-streaming clusters add external weight storage and gradient reduction.", "inference": "Serves supported models using distributed on-wafer SRAM; system partitioning and external networking still matter for larger deployments.", "components": [ "Wafer-scale processor", "Many compute cores", "Distributed SRAM", "On-wafer routing fabric", "External I/O", "Host and input servers", "External weight storage", "Broadcast and reduction network", "Dedicated power and cooling" ], "interactions": [ "The compiler distributes tensor operations across cores on one large processor.", "Nearby SRAM feeds compute; on-wafer routes carry intermediate tensors between cores.", "In Cerebras weight-streaming training, MemoryX holds weights and streams them toward the compute systems.", "SwarmX broadcasts weights and reduces returning gradients across a training cluster.", "Input and management servers prepare examples and coordinate execution; inference deployments can use different arrangements." ], "memoryModel": "Distributed on-wafer SRAM plus workload-specific external storage; training weight streaming and inference weight residency are distinct modes.", "interconnect": "Dense on-wafer fabric, external system links and, for the cited training cluster, SwarmX broadcast/reduction networking.", "bottlenecks": [ "On-wafer SRAM capacity", "External input or weight-stream bandwidth", "Mapping and supported operations", "Power delivery and cooling" ], "examples": [ "Cerebras WSE-3 and CS-3", "Cerebras CS-2 weight-streaming cluster" ], "visualNotes": "Unfold a system into one large wafer instead of many small GPUs. Zoom into a repeated core/SRAM tile; show external MemoryX only when the selected execution mode uses it.", "sourceIds": [ "hw-cerebras-cluster", "hw-cerebras-wse3", "hw-cerebras-inference" ], "history": { "id": "wafer-scale", "pioneers": [ { "name": "Cerebras Systems engineering team", "kind": "organisation", "contribution": "Modern wafer-scale AI processor implementation", "sourceIds": [ "hist-wse" ] } ], "milestones": [ { "date": "2019-08-19", "precision": "day", "type": "Public unveiling", "title": "First Cerebras Wafer Scale Engine unveiled", "description": "Cerebras publicly presents its 1.2-trillion-transistor WSE for AI computation.", "sourceIds": [ "hist-wse" ] } ], "note": "This dates Cerebras’s modern AI implementation. Wafer-scale integration had earlier research and commercial attempts; 2019 is not its original invention date." } }, { "id": "inference-asic", "name": "Inference-oriented ASIC", "category": "Specialized digital compute", "maturity": "Commercial cloud and edge products; a workload-specialization lens that overlaps systolic and dataflow designs.", "training": "Not a universal training exclusion: some underlying engines can support training, but product software and service support determine what is usable. Weights are often trained elsewhere.", "inference": "Designed around serving goals such as latency, throughput and energy; SRAM-centric deterministic engines and HBM-backed tensor engines are different variants.", "components": [ "Host CPU", "Specialized tensor or streaming engines", "Vector and scalar units", "On-chip SRAM", "HBM or other external memory where used", "DMA engines", "Device links", "Compiler and serving runtime" ], "interactions": [ "A compiler lowers a trained model to supported instructions and memory placements.", "The runtime submits requests; compute engines read weights, execute layers and maintain per-request state.", "In the cited Groq design, the compiler predetermines execution and data movement around SRAM-resident data.", "Inferentia2 uses HBM and NeuronLink, illustrating that inference ASICs do not share one memory topology.", "Multiple devices exchange activations or partial results when a model is partitioned." ], "memoryModel": "Variant-dependent: substantial local SRAM, HBM-backed hierarchies or other memories. SRAM capacity can force distribution across chips.", "interconnect": "Host I/O and product-specific chip links; AWS Inferentia2 uses NeuronLink-v2.", "bottlenecks": [ "Model/compiler compatibility", "Memory capacity and bandwidth", "Per-request state growth", "Communication and batch/latency tradeoffs" ], "examples": [ "AWS Inferentia2", "Groq LPU architecture" ], "visualNotes": "Offer two explicit variants: SRAM-centric streaming and HBM-backed tensor ASIC. Animate request state separately from fixed weights; avoid a universal inference-chip diagram.", "sourceIds": [ "hw-aws-inferentia2", "hw-aws-inferentia", "hw-groq-lpu", "hw-aws-glossary" ], "history": { "id": "inference-asic", "pioneers": [ { "name": "Google TPU team", "kind": "organisation", "contribution": "Large-scale deployment of a dedicated neural-network inference ASIC", "sourceIds": [ "hist-tpu" ] }, { "name": "Norman P. Jouppi and co-authors", "kind": "individuals", "contribution": "Published the first-generation TPU architecture and datacenter analysis", "sourceIds": [ "hist-tpu" ] } ], "milestones": [ { "date": "2015", "precision": "year", "type": "Deployment", "title": "Inference TPU deployed in datacenters", "description": "Google’s first-generation TPU accelerates neural-network inference; this overlaps the tensor-ASIC family.", "sourceIds": [ "hist-tpu" ] }, { "date": "2017", "precision": "year", "type": "Research publication", "title": "TPU architecture and performance analysis published", "description": "Jouppi and co-authors document the deployed design at ISCA.", "sourceIds": [ "hist-tpu" ] } ], "note": "A selected datacenter milestone, not the first neural-network chip ever built. Groq’s SRAM-centric card shown above is a different, later implementation." } }, { "id": "fpga-adaptive", "name": "FPGA and adaptive SoC", "category": "Reconfigurable digital compute", "maturity": "Established commercial hardware for custom pipelines and specialized inference.", "training": "Possible with custom designs, but the cited Versal products emphasize inference; general-purpose large-model training is not implied.", "inference": "Can combine preprocessing, quantized neural operators, postprocessing and unusual I/O into a tailored streaming pipeline.", "components": [ "Programmable logic", "Configurable routing", "DSP arithmetic blocks", "Block SRAM", "AI Engine tiles on supported devices", "Embedded CPU on SoCs", "DMA engines", "Memory controller", "External DRAM or HBM", "I/O transceivers" ], "interactions": [ "Hardware tools configure logic and routing; software separately programs hard processor and AI Engine blocks.", "Input streams flow through custom arithmetic pipelines with local buffering.", "AI Engine tiles communicate using local memory, DMA and stream links.", "A network-on-chip or programmable fabric connects tiles, external memory and I/O.", "The host loads model parameters and coordinates control; the configured pipeline processes successive inputs." ], "memoryModel": "Explicit local buffers and distributed SRAM backed by product-dependent external memory.", "interconnect": "Programmable wires, streaming interfaces, on-chip network, memory buses and external serial links.", "bottlenecks": [ "Logic, DSP and SRAM resource limits", "Place-and-route and compilation time", "External memory bandwidth", "Pipeline balancing and numerical precision" ], "examples": [ "AMD Versal AI Core adaptive SoC" ], "visualNotes": "Show routing rewiring around a custom pipeline, then expand hard AI Engine tiles separately. FPGA fabric and hard AI Engines should remain visibly different components.", "sourceIds": [ "hw-amd-versal-manual", "hw-amd-versal-product" ], "history": { "id": "fpga-adaptive", "pioneers": [ { "name": "Ross Freeman", "kind": "individual", "contribution": "FPGA invention and Xilinx co-founder", "sourceIds": [ "hist-fpga" ] }, { "name": "Xilinx", "kind": "organisation", "contribution": "Commercial XC2064 FPGA; now part of AMD", "sourceIds": [ "hist-fpga" ] } ], "milestones": [ { "date": "1985", "precision": "year", "type": "Commercial introduction", "title": "Xilinx XC2064", "description": "AMD’s historical account identifies the XC2064 as the first commercial FPGA.", "sourceIds": [ "hist-fpga" ] } ], "note": "The date marks commercial FPGA introduction. Programmable logic predates it; adaptive SoCs and hard AI Engine tiles were later developments." } }, { "id": "digital-near-memory", "name": "Digital near-memory and processing-in-memory", "category": "Compute-memory locality", "maturity": "Mixed: commercial PIM modules and research accelerators. The examples are not interchangeable or equally mature.", "training": "Selected operations may be offloaded near data; complete training capability depends on arithmetic, software and communication support.", "inference": "Targets reduced data movement, from embedding lookups near DRAM to neural inference with distributed local SRAM.", "components": [ "Host CPU", "Memory banks", "Digital processors near memory", "Local instruction and data storage", "Memory interfaces", "On-chip network where used", "Host orchestration software" ], "interactions": [ "The host partitions data and sends computation to the processors located beside that data.", "Local digital engines read nearby memory and compute without repeatedly moving the full data set to a distant CPU.", "The host or on-chip network combines results when work spans partitions.", "UPMEM embeds programmable DPUs in DRAM chips; cross-DPU data sharing requires explicit coordination.", "IBM NorthPole instead couples SRAM, compute and control in a distributed on-chip array for inference." ], "memoryModel": "A locality spectrum, not one mechanism: logic beside DRAM banks differs from SRAM distributed alongside compute. Ordinary caching alone is not this category's focus.", "interconnect": "Memory channels and host-mediated transfers for the UPMEM example; on-chip data/control networks for NorthPole.", "bottlenecks": [ "Cross-partition communication", "Supported arithmetic and instruction set", "Local memory capacity", "Host-to-memory transfer overhead" ], "examples": [ "UPMEM PIM DRAM modules", "IBM NorthPole research accelerator" ], "visualNotes": "Move a compute block next to a memory bank and shorten the data path. Provide separate DRAM-PIM and SRAM-near-compute variants; do not draw analog crossbars here.", "sourceIds": [ "hw-upmem-technology", "hw-upmem-embeddings", "hw-ibm-northpole" ], "history": { "id": "digital-near-memory", "pioneers": [ { "name": "UPMEM", "kind": "organisation", "contribution": "An early commercial DRAM-based processing-in-memory implementation", "sourceIds": [ "hist-pim" ] }, { "name": "Fabrice Devaux", "kind": "individual", "contribution": "Presented UPMEM’s PIM architecture at Hot Chips 31", "sourceIds": [ "hist-pim" ] } ], "milestones": [ { "date": "2019", "precision": "year", "type": "Technical presentation", "title": "UPMEM PIM at Hot Chips 31", "description": "UPMEM presents programmable processors embedded in DRAM and exposed through DIMM modules.", "sourceIds": [ "hist-pim" ] } ], "note": "This is a documented modern implementation milestone. Processing near or inside memory has a much longer research lineage; UPMEM is not credited with inventing the whole concept." } }, { "id": "analog-in-memory", "name": "Analog in-memory matrix compute", "category": "Compute-memory locality", "maturity": "Research and emerging specialized hardware; this record is grounded in IBM PCM prototypes, not a claim of broad deployment.", "training": "Hardware-aware training commonly runs digitally before weights are programmed. Reliable on-device weight updates remain a research area in the cited work.", "inference": "Programmed conductances perform matrix-vector operations in place; digital circuitry handles remaining operations and communication.", "components": [ "Resistive or phase-change memory crossbars", "Input drivers or DACs", "Analog accumulation paths", "ADCs", "Digital activation units", "Digital buffers", "Programming and calibration circuits", "Digital interconnect", "Host controller" ], "interactions": [ "Training-derived weights are encoded as conductances in memory devices.", "Input electrical signals pass through a crossbar; device conductances weight contributions that combine on output lines.", "Converters digitize outputs; digital units apply nonlinearities, scaling and other unsupported operations.", "Digital links move activations between tiles.", "Calibration and hardware-aware model preparation compensate for noise, drift and device variability." ], "memoryModel": "The devices storing weights also participate in arithmetic; activations, conversion and control still require peripheral circuitry and buffers.", "interconnect": "Analog crossbar lines locally; digital inter-tile and host connections externally.", "bottlenecks": [ "Conversion energy and precision", "Device noise and conductance drift", "Reliable weight programming and update endurance", "Mapping models to finite arrays" ], "examples": [ "IBM 64-core mixed-signal PCM inference chip" ], "visualNotes": "Animate inputs crossing a conductance grid, summed outputs and conversion back to digital values. Expose converter and calibration overhead; avoid representing the entire system as analog.", "sourceIds": [ "hw-ibm-analog-chip", "hw-ibm-analog-training", "hw-ibm-hardware-aware" ], "history": { "id": "analog-in-memory", "pioneers": [ { "name": "Bernard Widrow / Stanford Electronics Laboratories", "kind": "individual", "contribution": "Early adaptive neurons with conductance-storing chemical memistors", "sourceIds": [ "hist-widrow" ] }, { "name": "IBM Research", "kind": "organisation", "contribution": "Modern phase-change-memory analog AI research", "sourceIds": [ "hw-ibm-analog-chip" ] } ], "milestones": [ { "date": "1960-10", "precision": "month", "type": "Research report", "title": "ADALINE neuron using chemical memistors", "description": "An early analog learning-hardware precursor in which adjustable conductance represents a weight.", "sourceIds": [ "hist-widrow" ] }, { "date": "2023", "precision": "year", "type": "Research demonstration", "title": "IBM 64-tile analog AI chip", "description": "IBM reports an analog in-memory inference chip combining memory tiles with digital processing and conversion.", "sourceIds": [ "hw-ibm-analog-chip" ] } ], "note": "The memistor is a historical precursor, not the same device as a modern two-terminal memristor or phase-change-memory crossbar. No sole inventor of this broad family is implied." } }, { "id": "photonic-compute", "name": "Photonic matrix compute with electronic control", "category": "Alternative physical compute", "maturity": "Demonstrated processors and emerging platforms; photonic compute has a different deployment status from optical networking.", "training": "Optical backpropagation has been demonstrated in research. It should not be presented as established large-model training infrastructure.", "inference": "Research processors execute real neural workloads using optical linear algebra combined with electronic memory and nonlinear operations.", "components": [ "Light source", "Optical modulators", "Waveguides and interferometers", "Programmable photonic matrix", "Photodetectors", "Data converters", "Electronic memory", "Digital control and nonlinear units", "Calibration circuitry" ], "interactions": [ "Electronic values are encoded onto optical signals.", "A configured optical network transforms the signals to perform a matrix operation.", "Photodetectors and conversion circuitry return results to electronic form.", "Electronic units store state and execute nonlinear or unsupported operations before the next optical stage.", "Control and calibration maintain the required optical settings and numerical accuracy." ], "memoryModel": "Hybrid: electronic storage and buffering supply an optical arithmetic path; light transmission alone does not provide conventional persistent model memory.", "interconnect": "Waveguides inside the optical compute path plus electronic and package links; optical networking between accelerators is a separate layer.", "bottlenecks": [ "End-to-end conversion overhead", "Precision and optical noise", "Calibration and thermal sensitivity", "Electronic memory and unsupported operations" ], "examples": [ "Lightmatter processor in the 2025 Nature demonstration", "Research photonic meshes with in-situ backpropagation" ], "visualNotes": "Use light paths only inside the optical operation, with visible electronic boundaries. Include a clear switch between photonic arithmetic and optical transport so they cannot be confused.", "sourceIds": [ "hw-photonic-nature", "hw-lightmatter-compute", "hw-photonic-training" ], "history": { "id": "photonic-compute", "pioneers": [ { "name": "Demetri Psaltis and Nabil Farhat", "kind": "individuals", "contribution": "Early optical neural-network work at Caltech; Farhat on leave from Penn", "sourceIds": [ "hist-optical" ] }, { "name": "Yichen Shen, Nicholas C. Harris and collaborators", "kind": "individuals", "contribution": "Integrated coherent nanophotonic neural-network demonstration", "sourceIds": [ "hist-nanophotonic" ] } ], "milestones": [ { "date": "1985-02-01", "precision": "day", "type": "Research publication", "title": "Optical associative neural-memory schemes", "description": "Psaltis and Farhat publish optical implementations of a neural associative-memory model.", "sourceIds": [ "hist-optical" ] }, { "date": "2017", "precision": "year", "type": "Research publication", "title": "Deep learning with coherent nanophotonic circuits", "description": "An integrated photonic neural-network demonstration is published in Nature Photonics; the preprint appeared in 2016.", "sourceIds": [ "hist-nanophotonic" ] } ], "note": "Optical computing has earlier roots. These are selected neural-computing milestones; optical networking and photonic arithmetic have distinct histories." } }, { "id": "neuromorphic", "name": "Neuromorphic and event-driven spiking compute", "category": "Alternative execution model", "maturity": "Research systems and specialized applications; the cited Intel Loihi 2 is a research processor.", "training": "Can implement supported local learning rules or run networks trained offline. This differs from assuming unrestricted transformer backpropagation support.", "inference": "Processes sparse events in stateful neuron models; useful research applications include sensory processing, robotics and optimization.", "components": [ "Neuron compute cores", "Local synaptic memory", "Neuron state storage", "Learning engines", "Event routers", "Asynchronous on-chip network", "Host processors", "Sensor or event encoders", "Inter-chip event links" ], "interactions": [ "A sensor or encoder emits events rather than necessarily delivering a dense tensor every clock cycle.", "Routers deliver events to relevant cores; local synaptic weights modify neuron state.", "Neuron dynamics produce output events when the model's conditions are met.", "On supported models, local learning engines update synapses from activity and learning signals.", "Events traverse chip boundaries when a network spans multiple processors; host software configures and observes execution." ], "memoryModel": "Neuron state and synaptic weights reside close to event-processing cores; sparse activity reduces unnecessary work when the workload permits it.", "interconnect": "Event-routing on-chip fabric and inter-chip links, plus host management connections.", "bottlenecks": [ "Model-to-hardware mapping", "Event congestion and synchronization", "Synaptic memory capacity", "Encoding dense data and supported learning rules" ], "examples": [ "Intel Loihi 2", "Intel Kapoho Point research board" ], "visualNotes": "Show quiet cores between events, then sparse pulses and changing neuron state. Keep spiking computation distinct from ordinary dense neural-network diagrams and analog memory compute.", "sourceIds": [ "hw-intel-neuromorphic", "hw-intel-loihi2" ], "history": { "id": "neuromorphic", "pioneers": [ { "name": "Carver Mead", "kind": "individual", "contribution": "Foundational neuromorphic electronic-systems research", "sourceIds": [ "hist-mead" ] }, { "name": "California Institute of Technology", "kind": "organisation", "contribution": "Mead’s research institution", "sourceIds": [ "hist-mead" ] } ], "milestones": [ { "date": "1990-10", "precision": "month", "type": "Research publication", "title": "Neuromorphic electronic systems", "description": "Mead’s influential Proceedings of the IEEE paper articulates neuromorphic electronic-system design.", "sourceIds": [ "hist-mead" ] } ], "note": "1990 dates a foundational paper, not the first idea of a spiking neuron or the beginning of all brain-inspired computing. Intel Loihi is a later implementation." } } ], "systems": { "components": [ { "id": "cpu", "name": "CPU / host processor", "layer": "compute", "role": "General-purpose physical processor. Runs application logic and often coordinates attached accelerators.", "trainingRole": "Prepares batches and launches work; can also execute training or offloaded optimizer updates.", "inferenceRole": "Runs serving processes, preprocessing and models that suit CPU execution.", "examples": [ "Host CPU", "Integrated SoC CPU" ], "sourceIds": [ "sys-cuda", "sys-offload", "sys-vllm" ] }, { "id": "accelerator", "name": "Accelerator compute", "layer": "compute", "role": "Physical parallel arithmetic engine; the exact core design depends on the hardware architecture.", "trainingRole": "Executes forward, backward and optimizer kernels when supported.", "inferenceRole": "Executes model operators. Autoregressive transformer generation typically processes the prompt in prefill, then repeatedly decodes new tokens.", "examples": [ "GPU compute and tensor cores", "Tensor-processing ASIC", "Edge NPU" ], "sourceIds": [ "sys-cuda", "sys-autograd", "sys-edge", "sys-disagg" ] }, { "id": "device-memory", "name": "Device memory / HBM", "layer": "memory", "role": "Physical memory supplying accelerator data. HBM is one implementation; GDDR or shared system memory may be used instead.", "trainingRole": "Holds local weights, gradients, activations and optimizer allocations as capacity allows.", "inferenceRole": "Holds weights, temporary tensors and model-dependent caches.", "examples": [ "Stacked HBM", "GDDR", "Shared LPDDR on a SoC" ], "sourceIds": [ "sys-hbm", "sys-cuda", "sys-zero" ] }, { "id": "host-memory", "name": "Host DRAM", "layer": "memory", "role": "Physical CPU-accessible working memory; it may be shared with integrated accelerators.", "trainingRole": "Stages datasets and checkpoints; may hold offloaded model and optimizer state.", "inferenceRole": "Stages weights and requests; may hold offloaded data.", "examples": [ "DDR DIMMs", "LPDDR attached to CPU/SoC" ], "sourceIds": [ "sys-dgx-node", "sys-offload" ] }, { "id": "sram-cache", "name": "On-chip SRAM, cache and registers", "layer": "memory", "role": "Physical on-die storage close to arithmetic units; capacities and programmability differ by processor.", "trainingRole": "Reuses tensor tiles and intermediate values inside kernels.", "inferenceRole": "Keeps working tiles close to compute; distinct from the logical KV cache.", "examples": [ "GPU shared memory and L1/L2", "Registers", "Accelerator scratchpad" ], "sourceIds": [ "sys-cuda-kernels" ] }, { "id": "local-storage", "name": "Local flash / NVMe SSD", "layer": "storage", "role": "Physical persistent storage attached to a node.", "trainingRole": "Caches input data and checkpoints; optional offload tier.", "inferenceRole": "Stores model files and local data; loading is separate from steady-state execution.", "examples": [ "NVMe boot SSD", "NVMe dataset cache" ], "sourceIds": [ "sys-dgx-node", "sys-gds" ] }, { "id": "shared-storage", "name": "Shared storage service", "layer": "storage", "role": "An assembled persistent storage system, backed by drives and storage servers.", "trainingRole": "Supplies datasets and receives recovery checkpoints.", "inferenceRole": "Distributes model artifacts and stores retrieval content.", "examples": [ "Parallel filesystem", "Object storage", "Shared file storage" ], "sourceIds": [ "sys-superpod", "sys-retrieval" ] }, { "id": "pcie", "name": "PCIe root complex and switches", "layer": "interconnect", "role": "Physical I/O fabric connecting CPUs and attached devices; not the same as the accelerator scale-up fabric.", "trainingRole": "Carries host-device and storage transfers; topology affects peer access.", "inferenceRole": "Carries model loading and I/O; shared/coherent SoCs may use a different path.", "examples": [ "PCIe root complex", "PCIe switch" ], "sourceIds": [ "sys-rdma", "sys-gds" ] }, { "id": "nic", "name": "Network interface / RDMA adapter", "layer": "interconnect", "role": "Physical network endpoint connecting a compute node to an external fabric.", "trainingRole": "Transports synchronization and distributed tensor traffic.", "inferenceRole": "Transports requests and distributed inference data.", "examples": [ "Ethernet NIC with RoCE", "InfiniBand adapter" ], "sourceIds": [ "sys-rdma", "sys-dgx-network" ] }, { "id": "dpu", "name": "DPU / infrastructure offload processor", "layer": "interconnect", "role": "Optional physical processor combining networking and programmable infrastructure services.", "trainingRole": "Offloads selected networking, storage or security work from hosts.", "inferenceRole": "Supports networking/storage services and isolation; not a substitute for model compute.", "examples": [ "BlueField-class DPU", "Programmable SmartNIC" ], "sourceIds": [ "sys-dpu" ] }, { "id": "scale-up-switch", "name": "Scale-up switch / local accelerator fabric", "layer": "interconnect", "role": "Physical interconnect inside a fast accelerator domain; some designs use direct links without switches.", "trainingRole": "Moves tensors among closely coupled accelerators.", "inferenceRole": "Supports model shards that communicate during each forward pass.", "examples": [ "NVSwitch", "Direct accelerator links as an alternative" ], "sourceIds": [ "sys-dgx-network", "sys-tensor" ] }, { "id": "scale-out-switch", "name": "Scale-out network switch", "layer": "interconnect", "role": "Physical packet switch linking nodes or racks into a cluster.", "trainingRole": "Carries collective traffic between nodes.", "inferenceRole": "Carries service traffic, distributed model traffic and cache transfers.", "examples": [ "Ethernet leaf/spine switch", "InfiniBand switch" ], "sourceIds": [ "sys-superpod", "sys-optics" ] }, { "id": "optical-transceiver", "name": "Optical transceiver and fiber", "layer": "interconnect", "role": "Physical electrical-to-optical link endpoints plus fiber; short connections may use copper instead.", "trainingRole": "Carries network bits over optical links.", "inferenceRole": "Provides the same transport function for inference traffic.", "examples": [ "Pluggable optical module", "Active optical cable" ], "sourceIds": [ "sys-optics" ] }, { "id": "storage-fabric", "name": "Storage fabric", "layer": "interconnect", "role": "A storage-oriented network assembled from adapters, links and switches; it may be separate or shared.", "trainingRole": "Connects compute to shared training data and checkpoints.", "inferenceRole": "Connects services to model artifacts and retrieval stores.", "examples": [ "Dedicated storage Ethernet", "Storage InfiniBand" ], "sourceIds": [ "sys-superpod" ] }, { "id": "ups", "name": "UPS / backup energy", "layer": "power", "role": "Physical power-protection system with stored energy; facility topology and redundancy vary.", "trainingRole": "Supports continuity during upstream interruptions.", "inferenceRole": "Supports availability during upstream interruptions.", "examples": [ "Central UPS", "Distributed UPS and batteries" ], "sourceIds": [ "sys-ups" ] }, { "id": "pdu", "name": "Power distribution / PDU", "layer": "power", "role": "Physical distribution from facility circuits to IT loads; rack designs may use AC feeds or DC busbars.", "trainingRole": "Delivers electrical capacity to compute and networking.", "inferenceRole": "Delivers electrical capacity to inference servers.", "examples": [ "Rack PDU", "Remote power panel" ], "sourceIds": [ "sys-pdu" ] }, { "id": "psu", "name": "PSU / power shelf / voltage regulation", "layer": "power", "role": "Physical conversion and regulation from input power to the DC rails used by electronics.", "trainingRole": "Supplies compute rails under changing training load.", "inferenceRole": "Supplies rails under serving load.", "examples": [ "Server PSU", "Rack power shelf", "Board voltage regulators" ], "sourceIds": [ "sys-dgx-power" ] }, { "id": "cold-plate", "name": "Cold plates and liquid loop", "layer": "cooling", "role": "Physical direct-to-chip heat collection; this is an optional cooling design.", "trainingRole": "Removes heat from sustained accelerator and CPU activity.", "inferenceRole": "Removes heat from deployed compute.", "examples": [ "CPU/GPU cold plate", "Rack manifold" ], "sourceIds": [ "sys-dgx-rack" ] }, { "id": "cdu", "name": "Coolant distribution unit", "layer": "cooling", "role": "Physical pumping/control and heat-exchange equipment coupling cooling loops; designs vary.", "trainingRole": "Maintains thermal conditions for liquid-cooled compute.", "inferenceRole": "Maintains thermal conditions for liquid-cooled serving.", "examples": [ "Liquid-to-liquid CDU", "Liquid-to-air CDU" ], "sourceIds": [ "sys-liquid", "sys-cdu" ] }, { "id": "heat-rejection", "name": "Facility heat rejection", "layer": "cooling", "role": "Physical equipment transferring collected heat out of the IT cooling system.", "trainingRole": "Rejects heat produced by the training installation.", "inferenceRole": "Rejects heat produced by the serving installation.", "examples": [ "Dry cooler", "Chiller/condenser", "Cooling tower where used" ], "sourceIds": [ "sys-liquid", "sys-cdu" ] }, { "id": "air-cooling", "name": "Heat sinks, fans and room airflow", "layer": "cooling", "role": "Physical air path; it can cool an entire server or complement liquid cooling.", "trainingRole": "Removes heat from air-cooled devices.", "inferenceRole": "Cools server or edge components according to deployment.", "examples": [ "Heat sink and server fans", "Room air cooling" ], "sourceIds": [ "sys-dgx-node", "sys-dgx-rack" ] }, { "id": "package", "name": "Advanced package / chiplet interconnect", "layer": "packaging", "role": "Physical integration of compute dies, interconnect and possibly HBM. A package is a containment level, not another accelerator.", "trainingRole": "Provides die-to-die and memory connections used by training hardware.", "inferenceRole": "Provides the same physical integration for inference hardware.", "examples": [ "2.5D interposer", "Chiplet package", "3D stacking" ], "sourceIds": [ "sys-cowos" ] }, { "id": "foundry", "name": "Logic fabrication and packaging supply", "layer": "supply-chain", "role": "Upstream manufacturing capability, outside the running data center.", "trainingRole": "Enables production of training processors and switches.", "inferenceRole": "Enables production of inference processors and switches.", "examples": [ "Logic wafer fabrication", "Advanced packaging and test" ], "sourceIds": [ "sys-foundry" ] }, { "id": "memory-supply", "name": "Memory manufacturing supply", "layer": "supply-chain", "role": "Upstream DRAM/HBM manufacturing and stacking; not a runtime component.", "trainingRole": "Supplies capacity and bandwidth for training systems.", "inferenceRole": "Supplies memory used by serving systems.", "examples": [ "HBM stack manufacturing", "DRAM manufacturing" ], "sourceIds": [ "sys-hbm", "sys-cowos" ] }, { "id": "compiler-runtime", "name": "Compiler, runtime and device libraries", "layer": "software", "role": "Software translating operators into hardware execution and managing buffers and launches.", "trainingRole": "Schedules forward/backward execution and optimized kernels.", "inferenceRole": "Executes the supported inference graph and memory plan.", "examples": [ "XLA/PJRT", "GPU runtime and operator libraries" ], "sourceIds": [ "sys-xla", "sys-xla-gpu" ] }, { "id": "orchestration", "name": "Cluster orchestration and placement", "layer": "software", "role": "Software assigning jobs/services to available resources; distinct from token-level scheduling.", "trainingRole": "Allocates resources for distributed training workers.", "inferenceRole": "Places serving workers and replicas.", "examples": [ "Kubernetes with device plugins", "Batch scheduler" ], "sourceIds": [ "sys-kubernetes" ] }, { "id": "collectives", "name": "Collective communication library", "layer": "software", "role": "Software coordinating multi-device communication; not a physical network switch.", "trainingRole": "Combines gradients and gathers/shards tensors.", "inferenceRole": "Supports distributed model computations that require communication.", "examples": [ "NCCL all-reduce", "All-gather and reduce-scatter" ], "sourceIds": [ "sys-nccl" ] }, { "id": "inference-gateway", "name": "Inference API / gateway", "layer": "software", "role": "Software entry point for inference clients.", "trainingRole": "Not used in a basic training step; may serve evaluation or rollout requests.", "inferenceRole": "Accepts requests and returns streamed or complete results.", "examples": [ "HTTP inference API", "API server process" ], "sourceIds": [ "sys-vllm" ] }, { "id": "inference-scheduler", "name": "Inference request scheduler", "layer": "software", "role": "Software deciding which requests/tokens run next and coordinating workers.", "trainingRole": "Not used in a basic training step.", "inferenceRole": "Batches work and coordinates memory-aware execution.", "examples": [ "Continuous batching scheduler", "Engine core" ], "sourceIds": [ "sys-vllm" ] }, { "id": "kv-cache", "name": "KV cache", "layer": "logical-allocation", "role": "Logical tensor allocation containing past attention keys/values; not a separate memory chip and not universal to every model.", "trainingRole": "Usually absent from standard full-sequence training; useful in generation/rollout phases.", "inferenceRole": "Avoids recomputing cached attention state in autoregressive decoding.", "examples": [ "Paged KV blocks", "Prefix cache" ], "sourceIds": [ "sys-paged" ] }, { "id": "optimizer-state", "name": "Optimizer state", "layer": "logical-allocation", "role": "Logical training allocation, such as momentum or Adam moments; separate from parameters and gradients.", "trainingRole": "Read and updated by optimizer kernels; may be sharded or offloaded.", "inferenceRole": "Not used for ordinary inference with fixed weights.", "examples": [ "Adam moments", "Momentum buffers" ], "sourceIds": [ "sys-zero", "sys-offload" ] }, { "id": "model-state", "name": "Model parameters / weights", "layer": "logical-allocation", "role": "Logical learned tensor values stored in physical memory or persistent artifacts.", "trainingRole": "Read during model execution and changed by optimizer updates.", "inferenceRole": "Loaded and read during forward execution; normally unchanged.", "examples": [ "Weight tensors", "Quantized weight tensors" ], "sourceIds": [ "sys-fsdp", "sys-vllm" ] }, { "id": "activations", "name": "Activations and temporary tensors", "layer": "logical-allocation", "role": "Logical intermediate values produced by model operators.", "trainingRole": "Some are saved or recomputed for backward propagation.", "inferenceRole": "Temporary operator results; most do not persist across completed requests.", "examples": [ "Layer output", "Attention workspace" ], "sourceIds": [ "sys-autograd" ] }, { "id": "gradients", "name": "Gradients", "layer": "logical-allocation", "role": "Logical derivatives computed by backward propagation.", "trainingRole": "Feed optimizer updates and distributed synchronization.", "inferenceRole": "Not used in ordinary forward-only inference.", "examples": [ "Parameter gradients", "Activation gradients" ], "sourceIds": [ "sys-autograd", "sys-fsdp" ] }, { "id": "retrieval-service", "name": "Retrieval / embedding / reranking services", "layer": "software", "role": "Application-level pipeline retrieving relevant content for model input; not a hardware architecture.", "trainingRole": "May prepare or mine training data.", "inferenceRole": "Embeds queries, retrieves content and optionally reranks context.", "examples": [ "Vector search service", "Embedding model", "Reranker" ], "sourceIds": [ "sys-retrieval", "sys-retrieval-flow" ] }, { "id": "edge-io", "name": "Edge sensors and local input/output", "layer": "io", "role": "Physical local inputs and outputs around an edge compute system.", "trainingRole": "Can supply locally collected data when training is supported.", "inferenceRole": "Supplies camera/audio/user inputs and consumes local results.", "examples": [ "Camera", "Microphone", "Local display/controller" ], "sourceIds": [ "sys-edge" ] }, { "id": "bmc", "name": "Board management and telemetry", "layer": "management", "role": "Physical management controller and its firmware, separate from model execution.", "trainingRole": "Reports hardware health and supports node recovery.", "inferenceRole": "Reports hardware health and supports service recovery.", "examples": [ "BMC", "Out-of-band management port" ], "sourceIds": [ "sys-bmc" ] }, { "id": "federated-coordinator", "name": "Federated coordinator / aggregator", "layer": "software", "role": "Software coordinating learning across separate data owners or devices.", "trainingRole": "Distributes models and aggregates updates; keeping data local alone does not guarantee privacy.", "inferenceRole": "May distribute resulting models; not required for local inference.", "examples": [ "Federated averaging coordinator", "Secure aggregation service" ], "sourceIds": [ "sys-federated", "sys-fed-system" ] } ], "interactions": [ { "id": "read-local-data", "from": "local-storage", "to": "host-memory", "type": "data", "label": "Read local files", "training": "Stage training examples or checkpoints.", "inference": "Stage model files and inputs.", "transport": "NVMe / PCIe and host memory", "sourceIds": [ "sys-gds" ] }, { "id": "read-shared-data", "from": "shared-storage", "to": "storage-fabric", "type": "data", "label": "Read shared datasets or models", "training": "Read dataset shards.", "inference": "Read model artifacts or retrieval documents.", "transport": "Storage protocol over network", "sourceIds": [ "sys-superpod" ] }, { "id": "storage-to-host", "from": "storage-fabric", "to": "host-memory", "type": "data", "label": "Receive shared storage data", "training": "Fill host staging buffers.", "inference": "Fill model/input staging buffers.", "transport": "NIC and DMA; logical end-to-end path", "sourceIds": [ "sys-gds-overview" ] }, { "id": "write-checkpoint", "from": "host-memory", "to": "shared-storage", "type": "data", "label": "Persist checkpoint or artifact", "training": "Save recovery state; traffic travels through the storage fabric.", "inference": "Publish or cache model artifacts; not a token-step requirement.", "transport": "Storage fabric; logical end-to-end path", "sourceIds": [ "sys-superpod", "sys-checkpoint" ] }, { "id": "cpu-read-dram", "from": "host-memory", "to": "cpu", "type": "data", "label": "CPU reads working memory", "training": "Read samples, metadata and offloaded state.", "inference": "Read requests, tokens and model data.", "transport": "CPU memory channels", "sourceIds": [ "sys-cuda", "sys-offload" ] }, { "id": "cpu-write-dram", "from": "cpu", "to": "host-memory", "type": "data", "label": "CPU writes working memory", "training": "Prepare batches and optimizer results.", "inference": "Prepare inputs and output buffers.", "transport": "CPU memory channels", "sourceIds": [ "sys-cuda", "sys-offload" ] }, { "id": "host-to-device", "from": "host-memory", "to": "device-memory", "type": "data", "label": "Upload tensors", "training": "Move batches or offloaded values to the accelerator.", "inference": "Load weights or input tensors.", "transport": "PCIe or architecture-specific coherent link; may be shared memory", "sourceIds": [ "sys-cuda" ] }, { "id": "device-to-host", "from": "device-memory", "to": "host-memory", "type": "data", "label": "Download / offload tensors", "training": "Stage checkpoints or offloaded state.", "inference": "Return results or offload cache.", "transport": "PCIe or architecture-specific coherent link", "sourceIds": [ "sys-offload", "sys-disagg" ] }, { "id": "device-read", "from": "device-memory", "to": "accelerator", "type": "data", "label": "Read tensor operands", "training": "Read weights, activations and training state.", "inference": "Read weights, inputs and cache.", "transport": "Device memory interface", "sourceIds": [ "sys-cuda-kernels" ] }, { "id": "device-write", "from": "accelerator", "to": "device-memory", "type": "data", "label": "Write tensor results", "training": "Write activations, gradients and updated state.", "inference": "Write outputs and cache data.", "transport": "Device memory interface", "sourceIds": [ "sys-cuda-kernels" ] }, { "id": "cache-fill", "from": "device-memory", "to": "sram-cache", "type": "data", "label": "Bring a tile on chip", "training": "Reuse data within training kernels.", "inference": "Reuse data within forward kernels.", "transport": "On-chip memory hierarchy", "sourceIds": [ "sys-cuda-kernels" ] }, { "id": "cache-compute", "from": "sram-cache", "to": "accelerator", "type": "data", "label": "Feed near-compute operands", "training": "Supply kernel working values.", "inference": "Supply kernel working values.", "transport": "Register/cache/scratchpad datapath", "sourceIds": [ "sys-cuda-kernels" ] }, { "id": "pcie-control", "from": "cpu", "to": "pcie", "type": "control", "label": "Host manages attached devices", "training": "Coordinate device I/O.", "inference": "Coordinate device I/O.", "transport": "PCIe control and device drivers", "sourceIds": [ "sys-rdma" ] }, { "id": "pcie-device", "from": "pcie", "to": "accelerator", "type": "data", "label": "Carry attached-device traffic", "training": "Transport tensor data on supported paths.", "inference": "Transport input/model data on supported paths.", "transport": "PCIe; architecture-dependent", "sourceIds": [ "sys-gds" ] }, { "id": "direct-storage", "from": "local-storage", "to": "device-memory", "type": "data", "label": "Optional direct storage read", "training": "Read suitable input data without host bounce buffers.", "inference": "Read model or cache data where supported.", "transport": "GPUDirect Storage / DMA; requires compatible topology and software", "sourceIds": [ "sys-gds" ] }, { "id": "device-to-nic", "from": "device-memory", "to": "nic", "type": "network", "label": "Send device buffers", "training": "Send distributed tensors without host staging when supported.", "inference": "Send distributed tensors or KV state.", "transport": "Peer DMA / GPUDirect RDMA; hardware and software dependent", "sourceIds": [ "sys-rdma" ] }, { "id": "nic-to-device", "from": "nic", "to": "device-memory", "type": "network", "label": "Receive remote device buffers", "training": "Receive synchronized or partitioned tensors.", "inference": "Receive distributed tensors or transferred KV state.", "transport": "Peer DMA / GPUDirect RDMA; hardware and software dependent", "sourceIds": [ "sys-rdma" ] }, { "id": "nic-to-optics", "from": "nic", "to": "optical-transceiver", "type": "network", "label": "Encode network traffic on a link", "training": "Send cluster traffic.", "inference": "Send serving traffic.", "transport": "Electrical lanes into optical module; copper is an alternative", "sourceIds": [ "sys-optics" ] }, { "id": "optics-to-switch", "from": "optical-transceiver", "to": "scale-out-switch", "type": "network", "label": "Reach cluster switch", "training": "Carry inter-node traffic.", "inference": "Carry inter-node traffic.", "transport": "Fiber plus receiving transceiver", "sourceIds": [ "sys-optics" ] }, { "id": "switch-to-nic", "from": "scale-out-switch", "to": "nic", "type": "network", "label": "Deliver to destination node", "training": "Deliver distributed training traffic.", "inference": "Deliver serving/distributed inference traffic.", "transport": "Optical or copper link; summarized return path", "sourceIds": [ "sys-superpod" ] }, { "id": "scale-up-send", "from": "accelerator", "to": "scale-up-switch", "type": "network", "label": "Send inside fast device domain", "training": "Transfer layer shards, activations or reductions.", "inference": "Transfer tensors between model shards.", "transport": "NVLink-class fabric; vendor-specific alternatives exist", "sourceIds": [ "sys-dgx-network" ] }, { "id": "scale-up-receive", "from": "scale-up-switch", "to": "accelerator", "type": "network", "label": "Deliver to peer accelerator", "training": "Complete tightly coupled communication.", "inference": "Complete distributed forward-pass communication.", "transport": "NVLink-class fabric; separate peer instance", "sourceIds": [ "sys-dgx-network" ] }, { "id": "dpu-storage", "from": "dpu", "to": "storage-fabric", "type": "network", "label": "Offload storage networking", "training": "Handle selected storage transport services.", "inference": "Handle selected storage transport services.", "transport": "DPU network port / storage protocol", "sourceIds": [ "sys-dpu" ] }, { "id": "runtime-launch", "from": "compiler-runtime", "to": "accelerator", "type": "control", "label": "Launch compiled operators", "training": "Launch forward/backward/update kernels.", "inference": "Launch forward operators.", "transport": "Runtime API and device command queues", "sourceIds": [ "sys-xla", "sys-xla-gpu" ] }, { "id": "runtime-cpu", "from": "compiler-runtime", "to": "cpu", "type": "control", "label": "Execute host-side program", "training": "Run control code or CPU kernels.", "inference": "Run control code or CPU kernels.", "transport": "Compiled/native CPU execution", "sourceIds": [ "sys-xla" ] }, { "id": "place-workers", "from": "orchestration", "to": "compiler-runtime", "type": "control", "label": "Start workers on assigned resources", "training": "Launch workers on allocated devices.", "inference": "Launch serving workers/replicas.", "transport": "Scheduler and container/job lifecycle", "sourceIds": [ "sys-kubernetes" ] }, { "id": "request-collective", "from": "compiler-runtime", "to": "collectives", "type": "control", "label": "Invoke communication operation", "training": "Request gradient or state communication.", "inference": "Request distributed layer communication.", "transport": "Communication library API", "sourceIds": [ "sys-nccl", "sys-xla-gpu" ] }, { "id": "collective-device", "from": "collectives", "to": "accelerator", "type": "control", "label": "Coordinate communication kernels", "training": "Run all-reduce, all-gather or reduce-scatter.", "inference": "Run required distributed tensor operations.", "transport": "Library kernels mapped onto physical links", "sourceIds": [ "sys-nccl" ] }, { "id": "gateway-schedule", "from": "inference-gateway", "to": "inference-scheduler", "type": "control", "label": "Admit and route inference work", "training": "Not used in ordinary training; may serve rollouts.", "inference": "Send requests to an execution engine.", "transport": "In-process API, IPC or service transport", "sourceIds": [ "sys-vllm" ] }, { "id": "schedule-runtime", "from": "inference-scheduler", "to": "compiler-runtime", "type": "control", "label": "Dispatch the next batch", "training": "Not used in ordinary training.", "inference": "Choose work for the next engine iteration.", "transport": "Engine-to-worker dispatch", "sourceIds": [ "sys-vllm" ] }, { "id": "schedule-cache", "from": "inference-scheduler", "to": "kv-cache", "type": "control", "label": "Allocate and manage KV blocks", "training": "Not used in ordinary full-sequence training.", "inference": "Reserve/reuse/release logical cache allocations.", "transport": "Software block allocator; not a hardware cable", "sourceIds": [ "sys-paged" ] }, { "id": "kv-residency", "from": "kv-cache", "to": "device-memory", "type": "placement", "label": "Map KV tensors to physical memory", "training": "Not used in ordinary full-sequence training.", "inference": "Map logical cache blocks into device memory.", "transport": "Allocation/residency relationship, not an extra memory chip", "sourceIds": [ "sys-paged" ] }, { "id": "weights-residency", "from": "model-state", "to": "device-memory", "type": "placement", "label": "Map model tensors to memory", "training": "Place local weight shards.", "inference": "Place model weights or shards.", "transport": "Allocation/residency relationship", "sourceIds": [ "sys-fsdp" ] }, { "id": "optimizer-residency", "from": "optimizer-state", "to": "device-memory", "type": "placement", "label": "Map optimizer buffers to memory", "training": "Place training state; host/NVMe offload is optional.", "inference": "Not used in ordinary inference.", "transport": "Allocation/residency relationship", "sourceIds": [ "sys-zero" ] }, { "id": "forward-activations", "from": "accelerator", "to": "activations", "type": "data", "label": "Compute forward activations", "training": "Produce intermediate tensors.", "inference": "Produce temporary forward-pass results.", "transport": "Logical operator output stored in physical memory", "sourceIds": [ "sys-autograd" ] }, { "id": "backward-read", "from": "activations", "to": "accelerator", "type": "data", "label": "Read saved/recomputed activations", "training": "Supply backward computation.", "inference": "Not used for ordinary forward-only inference.", "transport": "Logical input to backward kernels", "sourceIds": [ "sys-autograd" ] }, { "id": "backward-gradients", "from": "accelerator", "to": "gradients", "type": "data", "label": "Compute gradients", "training": "Calculate derivatives in the backward pass.", "inference": "Not used in ordinary inference.", "transport": "Logical kernel output", "sourceIds": [ "sys-autograd" ] }, { "id": "gradient-sync", "from": "gradients", "to": "collectives", "type": "data", "label": "Synchronize gradient contributions", "training": "Combine contributions among training workers.", "inference": "Not used in ordinary inference.", "transport": "All-reduce or reduce-scatter through device/network fabrics", "sourceIds": [ "sys-fsdp" ] }, { "id": "optimizer-read", "from": "optimizer-state", "to": "accelerator", "type": "data", "label": "Read optimizer moments", "training": "Supply optimizer kernels; CPU execution is an alternative.", "inference": "Not used in ordinary inference.", "transport": "Logical tensor reads; physical location depends on offload", "sourceIds": [ "sys-zero", "sys-offload" ] }, { "id": "optimizer-write", "from": "accelerator", "to": "optimizer-state", "type": "data", "label": "Update optimizer moments", "training": "Persist revised optimizer buffers.", "inference": "Not used in ordinary inference.", "transport": "Logical tensor writes; physical location depends on offload", "sourceIds": [ "sys-zero" ] }, { "id": "update-weights", "from": "accelerator", "to": "model-state", "type": "data", "label": "Apply parameter update", "training": "Write weights using gradients and optimizer rules.", "inference": "Not used in ordinary fixed-weight inference.", "transport": "Optimizer kernels; CPU execution may be used instead", "sourceIds": [ "sys-offload" ] }, { "id": "read-weights", "from": "model-state", "to": "accelerator", "type": "data", "label": "Use learned parameters", "training": "Read weights for forward/backward operators.", "inference": "Read weights for prediction.", "transport": "Logical tensor reads through memory hierarchy", "sourceIds": [ "sys-fsdp" ] }, { "id": "prefill-kv", "from": "accelerator", "to": "kv-cache", "type": "data", "label": "Create or append attention state", "training": "Applicable to generation/rollouts, not standard full-sequence training.", "inference": "Prefill builds cache; decoding appends newly computed values.", "transport": "Logical attention-kernel output", "sourceIds": [ "sys-paged" ] }, { "id": "decode-kv", "from": "kv-cache", "to": "accelerator", "type": "data", "label": "Reuse prior attention state", "training": "Applicable to generation/rollouts.", "inference": "Decode reads earlier keys/values as required by the model.", "transport": "Logical attention-kernel input", "sourceIds": [ "sys-paged" ] }, { "id": "result-gateway", "from": "compiler-runtime", "to": "inference-gateway", "type": "data", "label": "Return inference results", "training": "May return evaluation/rollout results.", "inference": "Return generated tokens or prediction outputs.", "transport": "Worker/engine/API path, simplified end-to-end", "sourceIds": [ "sys-vllm" ] }, { "id": "retrieval-query", "from": "inference-gateway", "to": "retrieval-service", "type": "data", "label": "Look up relevant context", "training": "Not used in a basic training step.", "inference": "Submit a query for retrieval and reranking.", "transport": "Application/service call", "sourceIds": [ "sys-retrieval-flow" ] }, { "id": "retrieval-read", "from": "shared-storage", "to": "retrieval-service", "type": "data", "label": "Retrieve indexed content", "training": "Optional training data preparation.", "inference": "Read search candidates and their content.", "transport": "Database/storage query", "sourceIds": [ "sys-retrieval" ] }, { "id": "retrieval-context", "from": "retrieval-service", "to": "inference-gateway", "type": "data", "label": "Return grounding context", "training": "Not used in a basic training step.", "inference": "Return selected passages to augment model input.", "transport": "Application/service response", "sourceIds": [ "sys-retrieval-flow" ] }, { "id": "power-ups-pdu", "from": "ups", "to": "pdu", "type": "power", "label": "Distribute protected power", "training": "Supply electrical energy.", "inference": "Supply electrical energy.", "transport": "Electrical distribution; topology varies", "sourceIds": [ "sys-ups", "sys-pdu" ] }, { "id": "power-pdu-psu", "from": "pdu", "to": "psu", "type": "power", "label": "Feed server/rack conversion", "training": "Supply input power.", "inference": "Supply input power.", "transport": "AC feeds or system-specific distribution", "sourceIds": [ "sys-pdu" ] }, { "id": "power-compute", "from": "psu", "to": "accelerator", "type": "power", "label": "Supply regulated compute power", "training": "Energize accelerator rails.", "inference": "Energize accelerator rails.", "transport": "DC busbars and board regulators; simplified", "sourceIds": [ "sys-dgx-rack" ] }, { "id": "power-host", "from": "psu", "to": "cpu", "type": "power", "label": "Supply regulated host power", "training": "Energize CPU rails.", "inference": "Energize CPU rails.", "transport": "DC rails and board regulators; simplified", "sourceIds": [ "sys-dgx-rack" ] }, { "id": "heat-accelerator", "from": "accelerator", "to": "cold-plate", "type": "heat", "label": "Collect accelerator heat", "training": "Absorb heat when liquid cooling is used.", "inference": "Absorb heat when liquid cooling is used.", "transport": "Thermal interface and cold plate", "sourceIds": [ "sys-dgx-rack" ] }, { "id": "heat-cpu", "from": "cpu", "to": "air-cooling", "type": "heat", "label": "Collect heat through airflow", "training": "Example air-cooled CPU path; liquid cooling is an alternative.", "inference": "Example air-cooled CPU path; liquid cooling is an alternative.", "transport": "Heat sink and moving air", "sourceIds": [ "sys-dgx-node" ] }, { "id": "heat-liquid", "from": "cold-plate", "to": "cdu", "type": "heat", "label": "Carry heat through coolant", "training": "Return heated coolant to heat exchanger.", "inference": "Return heated coolant to heat exchanger.", "transport": "Secondary coolant loop; heat direction shown, liquid recirculates", "sourceIds": [ "sys-liquid" ] }, { "id": "heat-cdu-facility", "from": "cdu", "to": "heat-rejection", "type": "heat", "label": "Reject collected liquid-loop heat", "training": "Move heat toward the environment.", "inference": "Move heat toward the environment.", "transport": "Facility loop, air or refrigerant depending on design", "sourceIds": [ "sys-cdu" ] }, { "id": "heat-air-facility", "from": "air-cooling", "to": "heat-rejection", "type": "heat", "label": "Reject air-path heat", "training": "Move heat from air-cooled IT to facility cooling.", "inference": "Move heat from air-cooled IT to facility cooling.", "transport": "Room/rear-door cooling and heat rejection path", "sourceIds": [ "sys-liquid" ] }, { "id": "telemetry-control", "from": "bmc", "to": "orchestration", "type": "control", "label": "Report health to operations", "training": "Inform monitoring/recovery systems used alongside the scheduler.", "inference": "Inform monitoring/recovery systems used alongside the scheduler.", "transport": "Out-of-band management / Redfish; integration-dependent", "sourceIds": [ "sys-bmc" ] }, { "id": "manufacture-logic", "from": "foundry", "to": "package", "type": "supply", "label": "Supply fabricated logic and integration", "training": "Manufacturing dependency, not runtime traffic.", "inference": "Manufacturing dependency, not runtime traffic.", "transport": "Wafer/die production and packaging", "sourceIds": [ "sys-foundry" ] }, { "id": "manufacture-memory", "from": "memory-supply", "to": "package", "type": "supply", "label": "Supply memory stacks for integration", "training": "Manufacturing dependency for HBM packages.", "inference": "Manufacturing dependency for HBM packages.", "transport": "DRAM stacking and package assembly", "sourceIds": [ "sys-cowos", "sys-hbm" ] }, { "id": "integrate-compute", "from": "package", "to": "accelerator", "type": "supply", "label": "Integrate the processor package", "training": "Construction/containment relationship, not a runtime cable.", "inference": "Construction/containment relationship, not a runtime cable.", "transport": "Package assembly and die interconnect", "sourceIds": [ "sys-cowos" ] }, { "id": "edge-input", "from": "edge-io", "to": "cpu", "type": "data", "label": "Capture local inputs", "training": "Optional local data collection.", "inference": "Provide local sensor or user input.", "transport": "Platform I/O bus; deployment-specific", "sourceIds": [ "sys-edge" ] }, { "id": "edge-output", "from": "cpu", "to": "edge-io", "type": "data", "label": "Deliver local result", "training": "Optional feedback during local training.", "inference": "Display or act on local prediction.", "transport": "Platform I/O bus; deployment-specific", "sourceIds": [ "sys-edge" ] }, { "id": "federated-model", "from": "federated-coordinator", "to": "model-state", "type": "data", "label": "Distribute model to participants", "training": "Send selected round's parameters to clients.", "inference": "May distribute a trained model for local use.", "transport": "Service protocol over a wide-area network", "sourceIds": [ "sys-fed-system" ] }, { "id": "federated-update", "from": "model-state", "to": "federated-coordinator", "type": "data", "label": "Send local model update", "training": "Aggregate updates after local learning; raw examples can remain local.", "inference": "Not used in ordinary inference.", "transport": "Wide-area transport; privacy protections require explicit design", "sourceIds": [ "sys-fed-system" ] }, { "id": "optimizer-gradient", "from": "gradients", "to": "accelerator", "type": "data", "label": "Use synchronized gradients", "training": "Supply gradients to optimizer kernels.", "inference": "Not used in ordinary inference.", "transport": "Logical tensor reads; CPU optimizer is an alternative", "sourceIds": [ "sys-offload", "sys-fsdp" ] }, { "id": "activation-residency", "from": "activations", "to": "device-memory", "type": "placement", "label": "Map temporary tensors to memory", "training": "Store saved intermediates when not recomputed/offloaded.", "inference": "Store temporary operator values.", "transport": "Allocation/residency relationship", "sourceIds": [ "sys-autograd" ] }, { "id": "gradient-residency", "from": "gradients", "to": "device-memory", "type": "placement", "label": "Map gradient buffers to memory", "training": "Place local gradient tensors or shards.", "inference": "Not used in ordinary inference.", "transport": "Allocation/residency relationship", "sourceIds": [ "sys-fsdp" ] }, { "id": "power-memory", "from": "psu", "to": "device-memory", "type": "power", "label": "Supply memory power", "training": "Energize memory through package/board rails.", "inference": "Energize memory through package/board rails.", "transport": "DC rails; simplified physical path", "sourceIds": [ "sys-dgx-power" ] }, { "id": "power-network", "from": "psu", "to": "nic", "type": "power", "label": "Supply network-adapter power", "training": "Power network traffic handling.", "inference": "Power network traffic handling.", "transport": "Board/card electrical rails", "sourceIds": [ "sys-dgx-power" ] }, { "id": "power-switch", "from": "psu", "to": "scale-up-switch", "type": "power", "label": "Supply scale-up switching power", "training": "Power accelerator interconnect.", "inference": "Power accelerator interconnect.", "transport": "Switch power supply / rack busbar", "sourceIds": [ "sys-dgx-power" ] }, { "id": "heat-network", "from": "nic", "to": "air-cooling", "type": "heat", "label": "Collect network-device heat", "training": "Remove heat on air-cooled adapters.", "inference": "Remove heat on air-cooled adapters.", "transport": "Heat sinks and airflow; design-dependent", "sourceIds": [ "sys-dgx-rack" ] }, { "id": "power-cooling", "from": "pdu", "to": "cdu", "type": "power", "label": "Power liquid-cooling equipment", "training": "Supply pumps and cooling controls.", "inference": "Supply pumps and cooling controls.", "transport": "Electrical feed; equipment power conversion omitted", "sourceIds": [ "sys-cdu" ] } ], "patterns": [ { "id": "single-node", "name": "Single node / local execution", "workloads": [ "training", "inference" ], "description": "Keep the workload inside one host or device. A CPU-only run, one accelerator or several local accelerators are different hardware choices inside this deployment pattern. Multiple devices still need a partitioning or replication strategy. This is a deployment boundary, not a unique chip architecture.", "components": [ "cpu", "accelerator", "host-memory", "device-memory", "local-storage", "compiler-runtime", "activations", "gradients", "optimizer-state", "model-state" ], "interactions": [ "read-local-data", "cpu-read-dram", "host-to-device", "runtime-launch", "device-read", "device-write", "forward-activations", "backward-gradients", "optimizer-gradient", "optimizer-read", "optimizer-write", "update-weights" ], "bottleneck": "Local compute throughput, memory capacity/bandwidth and host-device movement; workload dependent.", "sourceIds": [ "sys-cuda", "sys-gds" ] }, { "id": "data-parallel", "name": "Replicated data-parallel training", "workloads": [ "training" ], "description": "Each worker has a model replica and processes different examples. Gradient synchronization keeps replicas aligned. Communication can overlap backward computation. This execution strategy can use CPUs or supported accelerators and can span a node or cluster.", "components": [ "accelerator", "device-memory", "gradients", "collectives", "nic", "scale-out-switch" ], "interactions": [ "backward-gradients", "gradient-sync", "collective-device", "device-to-nic", "switch-to-nic", "nic-to-device" ], "bottleneck": "Gradient communication, slow workers and the batch-size/convergence tradeoff.", "sourceIds": [ "sys-ddp" ] }, { "id": "sharded-data-parallel", "name": "Sharded state / FSDP / ZeRO", "workloads": [ "training" ], "description": "Partition parameters, gradients and/or optimizer state across data-parallel workers. Needed parameters are gathered for computation and gradients are reduced into shards. Sharding reduces per-device state residency at the cost of communication; it composes with other parallelism.", "components": [ "model-state", "gradients", "optimizer-state", "device-memory", "collectives", "accelerator", "compiler-runtime" ], "interactions": [ "weights-residency", "optimizer-residency", "request-collective", "gradient-sync", "collective-device" ], "bottleneck": "All-gather/reduce-scatter traffic and transient peak memory.", "sourceIds": [ "sys-fsdp", "sys-zero" ] }, { "id": "tensor-parallel", "name": "Tensor parallelism", "workloads": [ "training", "inference" ], "description": "Divide individual layer calculations across devices. Partial results communicate during model execution, making fast device links valuable. This partitions computation rather than adding another physical processor category.", "components": [ "accelerator", "device-memory", "collectives", "scale-up-switch", "compiler-runtime" ], "interactions": [ "request-collective", "collective-device", "scale-up-send", "scale-up-receive" ], "bottleneck": "Frequent communication latency and bandwidth on the critical path.", "sourceIds": [ "sys-tensor" ] }, { "id": "pipeline-parallel", "name": "Pipeline parallelism", "workloads": [ "training", "inference" ], "description": "Assign successive layer groups to different workers and pass activations between stages. Training also sends activation gradients backward. Multiple microbatches help fill the pipeline, but stage imbalance and empty pipeline slots waste capacity.", "components": [ "accelerator", "activations", "gradients", "nic", "scale-out-switch", "device-memory" ], "interactions": [ "forward-activations", "backward-read", "device-to-nic", "switch-to-nic", "nic-to-device" ], "bottleneck": "Pipeline bubbles, uneven stages and cross-stage transfer.", "sourceIds": [ "sys-megatron", "sys-vllm-dp" ] }, { "id": "expert-parallel", "name": "Expert parallelism / MoE routing", "workloads": [ "training", "inference" ], "description": "Place different expert subnetworks of a MoE model across devices. Tokens route to selected experts and results return to their originating computation. This is a model/execution choice that can share the same hardware used by dense models.", "components": [ "accelerator", "collectives", "scale-up-switch", "nic", "scale-out-switch", "device-memory", "compiler-runtime" ], "interactions": [ "request-collective", "scale-up-send", "scale-up-receive", "device-to-nic", "nic-to-device" ], "bottleneck": "All-to-all traffic, expert imbalance and routing overhead.", "sourceIds": [ "sys-megatron", "sys-vllm-dp" ] }, { "id": "context-parallel", "name": "Context / sequence partitioning", "workloads": [ "training", "inference" ], "description": "Partition sequence work or attention state across devices. Training can divide long-context computation; supported inference engines can shard the decode KV cache to reduce duplication. vLLM documents decode context parallelism for MLA and GQA models, while its prefill approaches remain under active development. Context and sequence parallelism are related but distinct; support depends on the model and backend.", "components": [ "accelerator", "activations", "device-memory", "collectives", "scale-up-switch", "compiler-runtime", "kv-cache" ], "interactions": [ "forward-activations", "request-collective", "collective-device", "scale-up-send", "scale-up-receive", "kv-residency", "decode-kv" ], "bottleneck": "Attention communication, training activation memory and inference KV-cache residency; additional sharding can increase communication overhead.", "sourceIds": [ "sys-megatron", "sys-tensor", "sys-vllm-context" ] }, { "id": "hybrid-parallel", "name": "Hybrid parallel execution", "workloads": [ "training", "inference" ], "description": "Combine partitioning dimensions: for example, tensor parallelism inside a fast domain and sharded data parallelism between hosts. MoE workloads may add expert parallelism. The selected dimensions and placement must suit the real topology.", "components": [ "accelerator", "collectives", "scale-up-switch", "nic", "scale-out-switch", "orchestration", "device-memory", "compiler-runtime" ], "interactions": [ "place-workers", "request-collective", "scale-up-send", "scale-up-receive", "device-to-nic", "nic-to-device" ], "bottleneck": "The slowest communication domain, placement quality and accumulated coordination costs.", "sourceIds": [ "sys-tensor", "sys-megatron" ] }, { "id": "inference-replicas", "name": "Inference replicas and batching", "workloads": [ "inference" ], "description": "Route requests among model-serving replicas; each replica can itself span multiple devices. Within an engine, scheduling batches work to use available compute and memory. Dense replicas need not synchronize gradients; some MoE deployments require coordination across ranks.", "components": [ "inference-gateway", "inference-scheduler", "compiler-runtime", "accelerator", "kv-cache", "device-memory" ], "interactions": [ "gateway-schedule", "schedule-runtime", "schedule-cache", "kv-residency", "runtime-launch", "result-gateway" ], "bottleneck": "Queueing latency, cache capacity, memory bandwidth and available parallel work.", "sourceIds": [ "sys-vllm-dp", "sys-vllm" ] }, { "id": "disaggregated-prefill-decode", "name": "Disaggregated prefill and decode", "workloads": [ "inference" ], "description": "Use different worker pools for prompt processing and token generation, transferring compatible KV state between them. Pools can use different parallelism settings. This is a serving pattern, not automatically different silicon; transfer overhead and workload determine usefulness. vLLM documents this feature as experimental.", "components": [ "inference-gateway", "inference-scheduler", "kv-cache", "device-memory", "nic", "scale-out-switch", "accelerator" ], "interactions": [ "gateway-schedule", "prefill-kv", "kv-residency", "device-to-nic", "switch-to-nic", "nic-to-device", "decode-kv" ], "bottleneck": "KV-transfer cost, pool imbalance and time-to-first-token versus inter-token-latency goals; no automatic throughput gain.", "sourceIds": [ "sys-disagg" ] }, { "id": "memory-tiering", "name": "Memory tiering and offload", "workloads": [ "training", "inference" ], "description": "Keep some tensors in host DRAM or storage and move them toward compute when needed. Training may offload optimizer state or parameters; inference can offload supported weights/cache. Extra capacity trades against movement cost and prefetch complexity. Offload support is software- and hardware-specific.", "components": [ "device-memory", "host-memory", "local-storage", "optimizer-state", "kv-cache", "cpu", "compiler-runtime", "accelerator" ], "interactions": [ "host-to-device", "device-to-host", "read-local-data", "direct-storage", "optimizer-read" ], "bottleneck": "PCIe/storage transfer bandwidth and latency, plus overlap with useful compute.", "sourceIds": [ "sys-offload", "sys-zero", "sys-gds", "sys-disagg" ] }, { "id": "edge-deployment", "name": "On-device / edge deployment", "workloads": [ "inference" ], "description": "Run a supported model near its input source using a CPU, integrated GPU or NPU. The hardware may share memory and have a much smaller power budget than a server. Local inference does not require a data center; on-device training is a separate capability, not assumed here.", "components": [ "edge-io", "cpu", "accelerator", "host-memory", "compiler-runtime" ], "interactions": [ "edge-input", "runtime-cpu", "runtime-launch", "edge-output" ], "bottleneck": "Power/thermal limits, memory capacity and supported operators.", "sourceIds": [ "sys-edge" ] }, { "id": "federated-learning", "name": "Federated learning", "workloads": [ "training" ], "description": "Coordinate local training across devices or organizations that retain their own examples, then aggregate updates. This is a data-placement and coordination pattern, not a chip type. Local data residency alone does not ensure privacy; secure aggregation and other protections are separate design choices.", "components": [ "federated-coordinator", "model-state", "cpu", "accelerator", "host-memory" ], "interactions": [ "federated-model", "read-weights", "update-weights", "federated-update" ], "bottleneck": "Participant availability, heterogeneous resources, non-identical data and wide-area communication.", "sourceIds": [ "sys-federated", "sys-fed-system" ] }, { "id": "retrieval-pipeline", "name": "Retrieval-augmented inference", "workloads": [ "inference" ], "description": "Retrieve relevant external content, optionally rerank it, then send selected context to a generative model. Embedding, search, reranking and generation can use different services and hardware. This is an application pipeline that composes with the serving patterns above.", "components": [ "inference-gateway", "retrieval-service", "shared-storage", "cpu", "accelerator", "inference-scheduler", "compiler-runtime" ], "interactions": [ "retrieval-query", "retrieval-read", "retrieval-context", "gateway-schedule", "result-gateway" ], "bottleneck": "Retrieval quality, index/storage latency, extra model calls and context length.", "sourceIds": [ "sys-retrieval", "sys-retrieval-flow" ] } ], "scales": [ { "id": "die", "name": "Die", "description": "One fabricated piece of silicon containing arithmetic units, on-chip memory and interfaces. Some accelerators span multiple dies; others use wafer-scale integration." }, { "id": "package", "name": "Package", "description": "One mounted component integrating one or more dies and potentially stacked memory. Distinguish physical containment from the logical data stored in memory." }, { "id": "board", "name": "Board / module", "description": "A circuit board or module bringing together packages, power regulation and physical connectors. Cards, accelerator baseboards and integrated SoC boards differ." }, { "id": "node", "name": "Compute node / server", "description": "A host or compute tray with processors, working memory, I/O and an operating-system boundary. It can contain one or several accelerators." }, { "id": "rack", "name": "Rack / scale-up domain", "description": "Mechanical enclosure with compute, switches, power and cooling. A fast communication domain may fit within a node, occupy a rack or span a different boundary; rack and domain are not synonyms." }, { "id": "cluster", "name": "Cluster", "description": "Networked compute nodes plus shared services running distributed jobs or serving replicas. Compute, storage and management traffic may use separate fabrics." }, { "id": "facility", "name": "Data center / facility", "description": "Building-level electrical, cooling and connectivity systems supporting one or more clusters. The data center is a deployment scale, not a processor architecture." }, { "id": "edge", "name": "Edge / end device", "description": "A deployment context near the input/user, from a phone or embedded device to a local server. It overlaps die-to-node scales rather than sitting above a facility." } ] }, "sources": [ { "id": "hw-intel-amx", "title": "What Is Intel Advanced Matrix Extensions (Intel AMX)?", "url": "https://www.intel.com/content/www/us/en/products/docs/accelerator-engines/what-is-intel-amx.html", "publisher": "Intel", "accessed": "2026-09-17" }, { "id": "hw-intel-ai-tuning", "title": "Tuning Guide for AI on the 4th Generation Intel Xeon Scalable Processors", "url": "https://www.intel.com/content/www/us/en/developer/articles/technical/tuning-guide-for-ai-on-the-4th-generation.html", "publisher": "Intel", "accessed": "2026-09-17" }, { "id": "hw-nvidia-blackwell", "title": "NVIDIA Blackwell Tuning Guide", "url": "https://docs.nvidia.com/cuda/blackwell-tuning-guide/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "hw-nvidia-system", "title": "System Hardware & Components — NVIDIA NVL72 AI Factory", "url": "https://docs.nvidia.com/enterprise-reference-architectures/nvl72-ai-factory/latest/components.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "hw-amd-mi300", "title": "AMD Instinct MI300 Series microarchitecture", "url": "https://rocmdocs.amd.com/en/latest/reference/gpu-arch/mi300.html", "publisher": "AMD", "accessed": "2026-09-17" }, { "id": "hw-apple-m4", "title": "Apple introduces M4 chip", "url": "https://www.apple.com/newsroom/2024/05/apple-introduces-m4-chip/", "publisher": "Apple", "accessed": "2026-09-17" }, { "id": "hw-apple-mlx", "title": "MLX documentation", "url": "https://ml-explore.github.io/mlx/build/html/index.html", "publisher": "Apple machine learning research / MLX contributors", "accessed": "2026-09-17" }, { "id": "hw-amd-mi300a", "title": "AMD Delivers Leadership Portfolio of Data Center AI Solutions with AMD Instinct MI300 Series", "url": "https://ir.amd.com/news-events/press-releases/detail/1173/amd-delivers-leadership-portfolio-of-data-center-ai-solutions-with-amd-instinct-mi300-series", "publisher": "AMD", "accessed": "2026-09-17" }, { "id": "hw-google-tpu", "title": "TPU architecture", "url": "https://docs.cloud.google.com/tpu/docs/system-architecture-tpu-vm", "publisher": "Google Cloud", "accessed": "2026-09-17" }, { "id": "hw-aws-trainium2", "title": "Trainium2 Architecture Guide for NKI", "url": "https://awsdocs-neuron.readthedocs-hosted.com/en/v2.31.1/nki/guides/architecture/trainium2_arch.html", "publisher": "AWS Neuron", "accessed": "2026-09-17" }, { "id": "hw-aws-glossary", "title": "Neuron Glossary", "url": "https://awsdocs-neuron.readthedocs-hosted.com/en/v2.28.1/about-neuron/arch/glossary.html", "publisher": "AWS Neuron", "accessed": "2026-09-17" }, { "id": "hw-sambanova-dataflow", "title": "Dataflow Architecture", "url": "https://sambanova.ai/products/dataflow-architecture", "publisher": "SambaNova", "accessed": "2026-09-17" }, { "id": "hw-sambanova-sn40l", "title": "SambaNova SN40L: Scaling the AI Memory Wall with Dataflow and Composition of Experts", "url": "https://arxiv.org/abs/2405.07518", "publisher": "Prabhakar et al. / MICRO 2024, author manuscript on arXiv", "accessed": "2026-09-17" }, { "id": "hw-sambanova-rdu", "title": "RDU: AI chips for inference at scale", "url": "https://sambanova.ai/products/rdu-ai-chips", "publisher": "SambaNova", "accessed": "2026-09-17" }, { "id": "hw-cerebras-cluster", "title": "Cerebras Wafer-Scale Cluster Overview, software version 1.9.1", "url": "https://training-api.cerebras.ai/en/1.9.1/wsc/cerebras-basics/how-cerebras-works.html", "publisher": "Cerebras", "accessed": "2026-09-17" }, { "id": "hw-cerebras-wse3", "title": "Cerebras announces third-generation Wafer Scale Engine", "url": "https://www.cerebras.ai/press-release/cerebras-announces-third-generation-wafer-scale-engine", "publisher": "Cerebras", "accessed": "2026-09-17" }, { "id": "hw-cerebras-inference", "title": "Disaggregated inference", "url": "https://www.cerebras.ai/blog/disaggregated-inference", "publisher": "Cerebras", "accessed": "2026-09-17" }, { "id": "hw-aws-inferentia2", "title": "Inferentia2 Architecture", "url": "https://awsdocs-neuron.readthedocs-hosted.com/en/v2.9.1/general/arch/neuron-hardware/inferentia2.html", "publisher": "AWS Neuron", "accessed": "2026-09-17" }, { "id": "hw-aws-inferentia", "title": "Amazon Inferentia", "url": "https://aws.amazon.com/ai/machine-learning/inferentia/", "publisher": "AWS", "accessed": "2026-09-17" }, { "id": "hw-groq-lpu", "title": "Real-time AI inference demand accelerates on GroqCloud", "url": "https://groq.com/newsroom/demand-for-real-time-ai-inference-from-groq-accelerates-week-over-week", "publisher": "Groq", "accessed": "2026-09-17" }, { "id": "hw-amd-versal-manual", "title": "Versal Adaptive SoC AI Engine Architecture Manual (AM009)", "url": "https://docs.amd.com/r/en-US/am009-versal-ai-engine", "publisher": "AMD", "accessed": "2026-09-17" }, { "id": "hw-amd-versal-product", "title": "Versal AI Core Series", "url": "https://www.amd.com/en/products/adaptive-socs-and-fpgas/versal/ai-core-series.html", "publisher": "AMD", "accessed": "2026-09-17" }, { "id": "hw-upmem-technology", "title": "UPMEM Technology", "url": "https://www.upmem.com/technology/", "publisher": "UPMEM", "accessed": "2026-09-17" }, { "id": "hw-upmem-embeddings", "title": "Offloading Embedding Lookups to Processing-In-Memory", "url": "https://www.upmem.com/wp-content/uploads/2024/10/240826-ABUMPIMP-2024-PIM-Rec.pdf", "publisher": "ABUMPIMP 2024 presentation hosted by UPMEM", "accessed": "2026-09-17" }, { "id": "hw-ibm-northpole", "title": "IBM NorthPole: An Architecture for Neural Network Inference with a 12nm Chip", "url": "https://research.ibm.com/publications/ibm-northpole-an-architecture-for-neural-network-inference-with-a-12nm-chip", "publisher": "IBM Research / ISSCC 2024", "accessed": "2026-09-17" }, { "id": "hw-ibm-analog-chip", "title": "An energy-efficient analog chip for AI inference", "url": "https://research.ibm.com/blog/analog-ai-chip-inference", "publisher": "IBM Research", "accessed": "2026-09-17" }, { "id": "hw-ibm-analog-training", "title": "New algorithms open possibilities for training AI models on analog chips", "url": "https://research.ibm.com/blog/analog-in-memory-training-algorithms", "publisher": "IBM Research", "accessed": "2026-09-17" }, { "id": "hw-ibm-hardware-aware", "title": "The marriage of training and inference for scaled deep learning analog hardware", "url": "https://research.ibm.com/publications/the-marriage-of-training-and-inference-for-scaled-deep-learning-analog-hardware", "publisher": "IBM Research / IEDM 2019", "accessed": "2026-09-17" }, { "id": "hw-photonic-nature", "title": "Universal photonic artificial intelligence acceleration", "url": "https://www.nature.com/articles/s41586-025-08854-x", "publisher": "Ahmed et al. / Nature", "accessed": "2026-09-17" }, { "id": "hw-lightmatter-compute", "title": "Photonic AI Acceleration: A New Kind of Computer", "url": "https://lightmatter.co/blog/a-new-kind-of-computer/", "publisher": "Lightmatter", "accessed": "2026-09-17" }, { "id": "hw-photonic-training", "title": "Experimentally realized in situ backpropagation for deep learning in nanophotonic neural networks", "url": "https://arxiv.org/abs/2205.08501", "publisher": "Pai et al., author manuscript on arXiv", "accessed": "2026-09-17" }, { "id": "hw-intel-neuromorphic", "title": "Neuromorphic Computing and Engineering with AI", "url": "https://www.intel.com/content/www/us/en/research/neuromorphic-computing.html", "publisher": "Intel Labs", "accessed": "2026-09-17" }, { "id": "hw-intel-loihi2", "title": "Taking Neuromorphic Computing with Loihi 2 to the Next Level", "url": "https://download.intel.com/newsroom/2021/new-technologies/neuromorphic-computing-loihi-2-brief.pdf", "publisher": "Intel Labs", "accessed": "2026-09-17" }, { "id": "sys-cuda", "title": "CUDA Programming Model", "url": "https://docs.nvidia.com/cuda/archive/13.1.0/cuda-programming-guide/01-introduction/programming-model.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-cuda-kernels", "title": "Writing CUDA SIMT Kernels", "url": "https://docs.nvidia.com/cuda/archive/13.1.0/cuda-programming-guide/02-basics/writing-cuda-kernels.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-dgx-node", "title": "Introduction to NVIDIA DGX B200 Systems", "url": "https://docs.nvidia.com/dgx/dgxb200-user-guide/introduction-to-dgxb200.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-dgx-rack", "title": "DGX Grace Blackwell Rack Hardware", "url": "https://docs.nvidia.com/dgx/dgxgb200-user-guide/hardware.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-dgx-network", "title": "DGX Grace Blackwell Networking", "url": "https://docs.nvidia.com/dgx/dgxgb200-user-guide/networking.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-superpod", "title": "DGX SuperPOD H100 Reference Architecture", "url": "https://docs.nvidia.com/dgx-superpod/reference-architecture-scalable-infrastructure-h100/latest/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-rdma", "title": "GPUDirect RDMA Overview", "url": "https://docs.nvidia.com/cuda/gpudirect-rdma/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-gds", "title": "GPUDirect Storage Design Guide", "url": "https://docs.nvidia.com/gpudirect-storage/design-guide/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-gds-overview", "title": "GPUDirect Storage Overview Guide", "url": "https://docs.nvidia.com/gpudirect-storage/overview-guide/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-optics", "title": "Layer 1 Data Center Cheat Sheet", "url": "https://docs.nvidia.com/networking-ethernet-software/knowledge-base/Setup-and-Getting-Started/layer-1-Data-Center-Cheat-Sheet/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-dpu", "title": "BlueField-2 InfiniBand/Ethernet DPU User Guide", "url": "https://docs.nvidia.com/nvidia-bluefield-2-infiniband-ethernet-dpu-user-guide.pdf", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-cowos", "title": "CoWoS Advanced Packaging", "url": "https://3dfabric.tsmc.com/english/dedicatedFoundry/technology/cowos.htm", "publisher": "TSMC", "accessed": "2026-09-17" }, { "id": "sys-foundry", "title": "HPC Platform and TSMC 3DFabric", "url": "https://www.tsmc.com/english/dedicatedFoundry/technology/platform_HPC_tech_WLSI", "publisher": "TSMC", "accessed": "2026-09-17" }, { "id": "sys-hbm", "title": "HBM3E Memory", "url": "https://www.micron.com/products/memory/hbm/hbm3e", "publisher": "Micron", "accessed": "2026-09-17" }, { "id": "sys-ups", "title": "UPS Fundamentals and FAQs", "url": "https://www.eaton.com/us/en-us/products/backup-power-ups-surge-it-power-distribution/backup-power-ups/uninterruptible-power-supply-faq.html", "publisher": "Eaton", "accessed": "2026-09-17" }, { "id": "sys-pdu", "title": "Rack PDU Buying Guide", "url": "https://www.eaton.com/us/en-us/products/backup-power-ups-surge-it-power-distribution/power-distribution-for-it-equipment/rack-pdu-buying-guide.html", "publisher": "Eaton", "accessed": "2026-09-17" }, { "id": "sys-liquid", "title": "Liquid Cooling for Data Centers", "url": "https://www.vertiv.com/49d8ee/globalassets/documents/white-papers/vertiv-liquid-cooling-wp-en-na-sl-70807-web_332687_0.pdf", "publisher": "Vertiv", "accessed": "2026-09-17" }, { "id": "sys-cdu", "title": "CoolPhase Coolant Distribution Unit", "url": "https://www.vertiv.com/en-us/products-catalog/thermal-management/high-density-solutions/vertiv-coolphase-cdu/", "publisher": "Vertiv", "accessed": "2026-09-17" }, { "id": "sys-xla", "title": "XLA Architecture", "url": "https://openxla.org/xla/architecture", "publisher": "OpenXLA", "accessed": "2026-09-17" }, { "id": "sys-xla-gpu", "title": "XLA GPU Architecture Overview", "url": "https://openxla.org/xla/gpu_architecture", "publisher": "OpenXLA", "accessed": "2026-09-17" }, { "id": "sys-kubernetes", "title": "Schedule GPUs", "url": "https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/", "publisher": "Kubernetes", "accessed": "2026-09-17" }, { "id": "sys-nccl", "title": "NCCL Collective Operations", "url": "https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/collectives.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-vllm", "title": "vLLM Architecture Overview", "url": "https://docs.vllm.ai/en/latest/design/arch_overview/", "publisher": "vLLM", "accessed": "2026-09-17" }, { "id": "sys-paged", "title": "vLLM Paged Attention", "url": "https://docs.vllm.ai/en/latest/design/paged_attention/", "publisher": "vLLM", "accessed": "2026-09-17" }, { "id": "sys-disagg", "title": "vLLM Disaggregated Prefilling (Experimental)", "url": "https://docs.vllm.ai/en/latest/features/disagg_prefill/", "publisher": "vLLM", "accessed": "2026-09-17" }, { "id": "sys-zero", "title": "DeepSpeed Zero Redundancy Optimizer", "url": "https://www.deepspeed.ai/tutorials/zero/", "publisher": "DeepSpeed", "accessed": "2026-09-17" }, { "id": "sys-offload", "title": "DeepSpeed ZeRO-Offload", "url": "https://www.deepspeed.ai/tutorials/zero-offload/", "publisher": "DeepSpeed", "accessed": "2026-09-17" }, { "id": "sys-autograd", "title": "PyTorch Autograd Mechanics", "url": "https://docs.pytorch.org/docs/stable/notes/autograd", "publisher": "PyTorch", "accessed": "2026-09-17" }, { "id": "sys-megatron", "title": "Megatron Core Parallelism Strategies Guide", "url": "https://docs.nvidia.com/megatron-core/developer-guide/latest/user-guide/parallelism-guide.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-tensor", "title": "PyTorch Large Scale Training with Tensor Parallel", "url": "https://docs.pytorch.org/tutorials/intermediate/TP_tutorial.html", "publisher": "PyTorch", "accessed": "2026-09-17" }, { "id": "sys-fsdp", "title": "PyTorch Fully Sharded Data Parallel (FSDP2)", "url": "https://docs.pytorch.org/tutorials/intermediate/FSDP_tutorial.html", "publisher": "PyTorch", "accessed": "2026-09-17" }, { "id": "sys-federated", "title": "TensorFlow Federated", "url": "https://www.tensorflow.org/federated", "publisher": "TensorFlow", "accessed": "2026-09-17" }, { "id": "sys-fed-system", "title": "Towards Federated Learning at Scale: System Design", "url": "https://arxiv.org/abs/1902.01046", "publisher": "Bonawitz et al. / Google", "accessed": "2026-09-17" }, { "id": "sys-edge", "title": "LiteRT On-Device Machine Learning Framework", "url": "https://developers.google.com/edge/litert", "publisher": "Google", "accessed": "2026-09-17" }, { "id": "sys-retrieval", "title": "NeMo Retriever", "url": "https://docs.nvidia.com/nemo/retriever/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-retrieval-flow", "title": "NeMo Retriever: Extraction, Embedding and Reranking", "url": "https://developer.nvidia.com/nemo-retriever", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-bmc", "title": "DGX B200 Redfish API Support", "url": "https://docs.nvidia.com/dgx/dgxb200-user-guide/redfish-api-supp.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-ddp", "title": "PyTorch Distributed Data Parallel Tutorial", "url": "https://docs.pytorch.org/tutorials/intermediate/ddp_tutorial.html", "publisher": "PyTorch", "accessed": "2026-09-17" }, { "id": "sys-vllm-dp", "title": "vLLM Data Parallel Deployment", "url": "https://docs.vllm.ai/en/latest/serving/data_parallel_deployment/", "publisher": "vLLM", "accessed": "2026-09-17" }, { "id": "sys-dgx-power", "title": "DGX GB200 SuperPOD Key Components", "url": "https://docs.nvidia.com/dgx-superpod/reference-architecture-scalable-infrastructure-gb200/latest/dgx-superpod-components.html", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "sys-vllm-context", "title": "vLLM Context Parallel Deployment", "url": "https://docs.vllm.ai/en/latest/serving/context_parallel_deployment/", "publisher": "vLLM", "accessed": "2026-09-17" }, { "id": "sys-checkpoint", "title": "PyTorch Saving and Loading Models and Training Checkpoints", "url": "https://docs.pytorch.org/tutorials/beginner/saving_loading_models.html", "publisher": "PyTorch", "accessed": "2026-09-17" }, { "id": "hist-manchester", "title": "The Manchester Small Scale Experimental Machine — The Baby", "url": "https://curation.cs.manchester.ac.uk/computer50/www.computer50.org/mark1/new.baby.html", "publisher": "University of Manchester", "accessed": "2026-09-17" }, { "id": "hist-cray", "title": "Cray history — Supercomputers inspired by curiosity", "url": "https://www.hpe.com/us/en/compute/hpc/cray.html", "publisher": "HPE / Cray", "accessed": "2026-09-17" }, { "id": "hist-nvidia", "title": "NVIDIA History — A Timeline of Innovation", "url": "https://www.nvidia.com/en-gb/about-nvidia/corporate-timeline/", "publisher": "NVIDIA", "accessed": "2026-09-17" }, { "id": "hist-apple-a11", "title": "The future is here: iPhone X", "url": "https://www.apple.com/newsroom/2017/09/the-future-is-here-iphone-x/", "publisher": "Apple", "accessed": "2026-09-17" }, { "id": "hist-systolic", "title": "Systolic Arrays for (VLSI), April 1978 report", "url": "https://www.eecs.harvard.edu/htk/static/files/1978-cmu-cs-report-kung-leiserson.pdf", "publisher": "H. T. Kung and Charles E. Leiserson / Carnegie Mellon", "accessed": "2026-09-17" }, { "id": "hist-tpu", "title": "In-Datacenter Performance Analysis of a Tensor Processing Unit", "url": "https://research.google/pubs/in-datacenter-performance-analysis-of-a-tensor-processing-unit/", "publisher": "Google Research / Jouppi et al.", "accessed": "2026-09-17" }, { "id": "hist-dataflow", "title": "A preliminary architecture for a basic data-flow processor", "url": "https://doi.org/10.1145/642089.642111", "publisher": "Jack B. Dennis and David P. Misunas / ACM", "accessed": "2026-09-17" }, { "id": "hist-sn40l", "title": "SambaNova Unveils New AI Chip, the SN40L", "url": "https://sambanova.ai/press/sambanova-unveils-new-chip-the-sn40l", "publisher": "SambaNova", "accessed": "2026-09-17" }, { "id": "hist-wse", "title": "Cerebras Systems Unveils the Industry’s First Trillion Transistor Chip", "url": "https://www.cerebras.ai/press-release/cerebras-systems-unveils-the-industrys-first-trillion-transistor-chip", "publisher": "Cerebras", "accessed": "2026-09-17" }, { "id": "hist-fpga", "title": "From Invention to AI Acceleration: Celebrating 40 Years of FPGA Innovation", "url": "https://www.amd.com/en/blogs/2025/from-invention-to-ai-acceleration--celebrating-40-years-of-fpga-.html", "publisher": "AMD / Xilinx", "accessed": "2026-09-17" }, { "id": "hist-pim", "title": "The true Processing In Memory accelerator — Hot Chips 31", "url": "https://old.hotchips.org/hc31/HC31_1.4_UPMEM.FabriceDevaux.v2_1.pdf", "publisher": "Fabrice Devaux / UPMEM", "accessed": "2026-09-17" }, { "id": "hist-widrow", "title": "An Adaptive ADALINE Neuron Using Chemical Memistors", "url": "https://www-isl.stanford.edu/~widrow/papers/t1960anadaptive.pdf", "publisher": "Bernard Widrow / Stanford Electronics Laboratories", "accessed": "2026-09-17" }, { "id": "hist-optical", "title": "Optical information processing based on an associative-memory model of neural nets with thresholding and feedback", "url": "https://opg.optica.org/ol/abstract.cfm?uri=ol-10-2-98", "publisher": "Demetri Psaltis and Nabil Farhat / Optics Letters", "accessed": "2026-09-17" }, { "id": "hist-nanophotonic", "title": "Deep learning with coherent nanophotonic circuits", "url": "https://www.nature.com/articles/nphoton.2017.93", "publisher": "Yichen Shen, Nicholas C. Harris et al. / Nature Photonics", "accessed": "2026-09-17" }, { "id": "hist-mead", "title": "Neuromorphic electronic systems", "url": "https://authors.library.caltech.edu/records/j6gtc-ptx47", "publisher": "Carver Mead / Caltech; Proceedings of the IEEE", "accessed": "2026-09-17" } ], "economics": { "schema_version": "0.1.0", "status": "deferred_policy_only", "policy_date": "2026-09-17", "description": "Future economic mapping policy for the AI infrastructure atlas. No transaction research or event collection has been performed in this phase.", "events": [], "coverage": { "industry_scope": "Organizations materially involved in AI compute, memory, interconnect, storage, systems, facilities, power, cooling, infrastructure software, and enabling supply chains.", "scope_claim": "Versioned educational industry map, not an exhaustive vendor or transaction census.", "recent_window_start": null, "recent_window_end": null, "window_status": "Needs definition before collection. Do not silently assume a date range.", "geography": "global_proposed" }, "funding_policy": { "source": "User instruction in this task", "comparison": "strictly_greater_than", "currency": "USD", "currency_status": "Assumption from dollar-denominated thresholds; confirm before collection.", "amount_basis": "Incremental equity proceeds in a specifically identified funding round, excluding separately stated debt, cumulative historical funding, and valuation.", "amount_basis_status": "Proposed normalization rule; mixed or ambiguous financing requires review.", "thresholds": [ { "stage": "seed", "amount_usd": 10000000, "operator": ">" }, { "stage": "series_a", "amount_usd": 20000000, "operator": ">" }, { "stage": "series_b", "amount_usd": 30000000, "operator": ">" }, { "stage": "series_c", "amount_usd": 50000000, "operator": ">" }, { "stage": "series_d", "amount_usd": 100000000, "operator": ">" } ], "boundary_rule": "An amount exactly equal to its threshold does not qualify under the user's more-than wording.", "unspecified_series": { "status": "needs_clarification", "threshold_usd": null, "note": "The user mentioned more than $10 million in Series, or seed. Preserve the ambiguity for unspecified Series rounds; do not apply the seed threshold to every Series round. Explicit A/B/C/D thresholds take precedence." }, "other_stages": { "pre_seed": "needs_policy", "series_e_or_later": "needs_policy", "growth": "needs_policy", "bridge": "needs_policy", "extension": "Link to the canonical round, deduplicate announcements, and review amount basis before qualification.", "unknown": "needs_clarification" }, "evaluation_order": [ "Confirm that the event is a funding round and that the subject organization is in scope.", "Record the source's stage verbatim and normalize only when the stage is supported by evidence.", "Identify whether reported proceeds are incremental, cumulative, equity, debt, mixed, or undisclosed.", "If equity proceeds cannot be separated from debt, return needs_review rather than comparing the combined total.", "If the source currency is not USD, retain the original amount and use a documented conversion with date and source; otherwise return needs_review.", "If stage, amount basis, or conversion is unresolved, return needs_review.", "For an exact disclosed amount in a configured stage, qualify only if normalized equity round proceeds are strictly greater than that stage threshold.", "For an approximate amount, range, or lower bound, retain its qualifier and require review; do not round it across the threshold.", "Merge duplicate coverage and link extensions/status updates before counting events." ], "outcomes": [ "qualifies", "below_or_equal_threshold", "needs_review", "out_of_scope" ], "unknown_value": "null; never replace undisclosed with zero" }, "non_funding_policy": { "apply_funding_thresholds": false, "mergers_and_acquisitions": { "significance_rule": "deferred_for_definition", "review_dimensions": [ "transaction value and its basis", "ownership/control change", "critical technology or supply-chain access", "market structure", "affected capacity or customers" ], "note": "Do not assume that equity-round thresholds apply to M&A; distinguish enterprise value, equity consideration, earnouts, and undisclosed amounts." }, "partnerships": { "significance_rule": "deferred_for_definition", "review_dimensions": [ "binding financial commitment", "capacity or purchase volume", "exclusivity", "joint product or integration", "manufacturing/supply commitment", "material deployment access" ], "note": "Distinguish signed commercial commitments from nonbinding memoranda, technical integrations, and promotional announcements. Do not infer financial value." }, "other_economic_events": { "event_types": [ "capacity_contract", "capital_expenditure_commitment", "joint_venture", "asset_sale", "debt_financing", "grant" ], "significance_rule": "deferred_for_definition" } }, "provenance_and_deduplication": { "source_priority": [ "regulatory filing or executed/public contract", "company or counterparty announcement", "investor announcement", "reputable reporting with attribution" ], "preserve_disagreements": true, "required_dates": [ "announced_at when known", "effective_or_closed_at when known", "source_published_at", "retrieved_at", "last_verified_at" ], "date_precision": [ "day", "month", "year", "unknown" ], "canonical_event_rule": "One underlying transaction or round has one event ID. Articles, announcement updates, and closing confirmations are sources or status history, not additional transactions.", "candidate_duplicate_key": [ "normalized participant organization IDs", "event type", "round or agreement identifier", "announcement date proximity" ], "candidate_key_is_not_automatic_proof": true, "round_extensions": "Record incremental proceeds and parent round ID; avoid summing a cumulative restatement and its previous announcements.", "event_statuses": [ "rumored", "announced", "signed", "pending_close", "closed", "cancelled", "terminated", "unknown" ], "publication_rule": "Confirmed and reported claims must carry their status and direct supporting sources. Rumors are not displayed as completed transactions." }, "future_field_schema": { "notation": "Field descriptors for the later database; this object contains no actual organizations or transactions.", "organization": { "id": "stable string", "legal_name": "string", "display_name": "string", "aliases": "array", "canonical_domain": "string|null", "organization_type": "company|investor|university|public_body|consortium|other", "headquarters_country": "ISO 3166-1 alpha-2|null", "parent_organization_id": "organization.id|null", "source_ids": "array", "last_verified_at": "ISO 8601 datetime" }, "organization_component_link": { "id": "stable string", "organization_id": "organization.id", "component_id": "atlas component.id", "role": "designer|manufacturer|packager|supplier|operator|software_provider|integrator|customer|other", "product_or_service": "string|null", "valid_from": "date|null", "valid_to": "date|null", "source_ids": "array", "confidence": "confirmed|reported|inferred", "inference_note": "string|null; required when inferred" }, "event": { "id": "stable string", "event_type": "funding_round|merger|acquisition|partnership|capacity_contract|capital_expenditure_commitment|joint_venture|asset_sale|debt_financing|grant", "title": "string", "summary": "string; distinguish sourced fact from interpretation", "status": "provenance_and_deduplication.event_statuses value", "announced_at": "date|null", "announcement_date_precision": "day|month|year|unknown", "effective_or_closed_at": "date|null", "close_date_precision": "day|month|year|unknown", "stage_as_reported": "string|null", "stage_normalized": "seed|series_a|series_b|series_c|series_d|other|unknown|null", "round_or_agreement_identifier": "string|null", "parent_event_id": "event.id|null; for extension or related transaction", "money_record_ids": "array", "source_ids": "array; direct support required", "significance_decision": "qualifies|below_or_equal_threshold|needs_review|out_of_scope|policy_pending", "significance_reason": "string", "policy_version": "string", "confidence": "confirmed|reported|unverified", "last_verified_at": "ISO 8601 datetime", "status_history": "array<{status, date, date_precision, source_ids}>" }, "event_participant": { "event_id": "event.id", "organization_id": "organization.id", "role": "recipient|lead_investor|investor|acquirer|target|merger_party|partner|supplier|customer|lender|grantor|other", "source_ids": "array" }, "money_record": { "id": "stable string", "event_id": "event.id", "amount_as_reported": "string|null; preserve source wording", "amount": "number|null", "currency": "ISO 4217|null", "disclosure": "disclosed|undisclosed|estimated|reported_unconfirmed", "qualifier": "exact|approximately|more_than|at_least|up_to|range|unknown", "lower_bound": "number|null", "upper_bound": "number|null", "capital_type": "equity|debt|convertible|mixed|grant|purchase_commitment|transaction_consideration|valuation|other|unknown", "amount_basis": "incremental_round|cumulative_round|cumulative_company_funding|equity_value|enterprise_value|contract_value|facility_limit|annual_spend|multi_year_commitment|valuation|unknown", "primary_or_secondary": "primary|secondary|mixed|not_applicable|unknown", "amount_usd": "number|null", "fx_rate_to_usd": "number|null; USD per source-currency unit", "fx_rate_date": "date|null", "fx_source_id": "source.id|null", "source_ids": "array", "qualification_eligible": "boolean; false for valuation, cumulative company funding, unseparated mixed capital, or unresolved amounts" }, "event_component_link": { "event_id": "event.id", "component_id": "atlas component.id", "architecture_ids": "array", "relationship": "funds_development|changes_ownership|expands_supply|expands_capacity|integrates_with|enables_access|other", "explanation": "string", "source_ids": "array", "confidence": "confirmed|reported|inferred", "inference_note": "string|null; required when inferred" }, "source": { "id": "stable string", "url": "absolute URL", "title": "string", "publisher": "string", "source_type": "filing|contract|company_announcement|counterparty_announcement|investor_announcement|reporting|other", "published_at": "date|null", "retrieved_at": "ISO 8601 datetime", "claim_locator": "page, section, paragraph, or table description", "supported_claims": "array; concise paraphrases", "correction_or_retraction": "string|null" } }, "open_decisions_before_collection": [ "Define the calendar window meant by recent.", "Confirm USD as the threshold currency and the conversion convention for non-USD announcements.", "Clarify the threshold for the user's unspecified Series wording and for stages outside seed through Series D.", "Confirm treatment of convertible rounds, secondary sales, extensions, and mixed equity/debt announcements.", "Define materiality criteria for M&A, partnerships, and other economic events independently of funding-round thresholds." ] }, "workflows": { "training": [ { "id": "load", "name": "Load a batch", "focusComponent": "local-storage", "description": "Examples move from storage into host memory. CPUs prepare a batch, then inputs reach accelerator memory. Direct storage-to-device paths are an optional alternative. Model weights are loaded before the loop; subsequent batches normally reuse resident weights.", "sourceIds": [ "sys-gds", "sys-cuda" ] }, { "id": "forward", "name": "Forward", "focusComponent": "accelerator", "description": "The accelerator reads weights and inputs, reuses tiles in on-chip storage, and computes predictions. Training saves selected intermediate activations for the backward pass, or recomputes some later to save memory.", "sourceIds": [ "sys-autograd", "sys-cuda-kernels" ] }, { "id": "loss", "name": "Compute loss", "focusComponent": "activations", "description": "An objective compares the model output with its training target or other learning signal. The resulting loss seeds differentiation. Labels and objectives vary across supervised, self-supervised, and other gradient-based training methods.", "sourceIds": [ "sys-autograd" ] }, { "id": "backward", "name": "Backward", "focusComponent": "gradients", "description": "Automatic differentiation applies the chain rule through the computation graph. Saved or recomputed activations help produce gradients. These are tensors stored in memory, not an extra physical chip.", "sourceIds": [ "sys-autograd" ] }, { "id": "synchronize", "name": "Synchronize", "focusComponent": "nic", "description": "Distributed workers communicate as the selected parallelism requires. Replicated data parallelism combines gradients; sharded approaches use different collective sequences. Communication can overlap backward computation. A single-device run skips this inter-device step.", "sourceIds": [ "sys-ddp", "sys-fsdp" ] }, { "id": "update", "name": "Update weights", "focusComponent": "optimizer-state", "description": "The optimizer uses gradients and any optimizer state to update weights. Adam-like methods retain additional tensors; other optimizers differ. State may be sharded or offloaded. Clearing or accumulating gradients prepares the next iteration.", "sourceIds": [ "sys-zero", "sys-offload" ] }, { "id": "checkpoint", "name": "Checkpoint", "focusComponent": "shared-storage", "description": "Periodically, the system persists a recovery checkpoint, including the required model and training state. Checkpointing is not necessarily performed after every batch. The next training step repeats the loop with updated weights.", "sourceIds": [ "sys-checkpoint" ] } ], "inference": [ { "id": "prepare", "name": "Load model", "focusComponent": "device-memory", "description": "A serving worker loads trained weights and prepares execution resources. This is typically worker startup or model switching, not a full weight reload for each request. This walkthrough uses an autoregressive transformer; other model families have different flows.", "sourceIds": [ "sys-vllm", "sys-cuda" ] }, { "id": "route", "name": "Route request", "focusComponent": "inference-gateway", "description": "The serving frontend receives a request, validates it, and prepares model inputs such as token IDs. A scheduler admits work according to capacity and batching policy. Retrieval can add context before model execution when the application uses it.", "sourceIds": [ "sys-vllm", "sys-retrieval" ] }, { "id": "prefill", "name": "Prefill", "focusComponent": "accelerator", "description": "The model processes prompt tokens and builds attention key/value state. Prompt processing exposes substantial parallel work. Actual limits depend on prompt length, batching, model shape, precision, and memory traffic; prefill is not universally compute-bound.", "sourceIds": [ "sys-paged", "sys-disagg" ] }, { "id": "decode", "name": "Decode", "focusComponent": "kv-cache", "description": "Each generation step uses model weights and cached attention keys/values to produce the next token. The cache grows with the sequence. Low-batch decode often pressures memory bandwidth; batching and model architecture can change the bottleneck.", "sourceIds": [ "sys-paged", "sys-vllm" ] }, { "id": "serve", "name": "Schedule + stream", "focusComponent": "inference-scheduler", "description": "The server can interleave active sequences, admit new work, and stream generated tokens. A disaggregated deployment may move KV state from prefill workers to decode workers; that adds network and coordination costs and is optional.", "sourceIds": [ "sys-vllm", "sys-disagg" ] }, { "id": "complete", "name": "Finish + release", "focusComponent": "kv-cache", "description": "At completion or cancellation, the serving system releases request resources, subject to prefix-cache retention policies. Ordinary inference keeps trained weights unchanged: it does not perform a training backward pass or optimizer update.", "sourceIds": [ "sys-paged", "sys-vllm" ] } ] }, "architectureLinks": { "note": "Generic, optional teaching connections between hardware families and the component catalog; these are not bills of materials or claims that every product contains every linked component. The generic accelerator box represents the selected alternative compute family, not an additional GPU; the CPU-only row therefore omits it. Components can be integrated in one package or share physical memory. Model state, activations, gradients, optimizer state and KV cache are logical allocations, not separate chips. Training allocations apply only to supported training paths, and KV cache applies only to models using attention caching. Facility components apply to relevant deployments, software links do not assert support for a particular framework, and supply-chain links sit outside runtime data flow. Family-specific mechanisms remain defined in architectures.json; this join does not replace them.", "links": [ { "architectureId": "cpu", "componentIds": [ "cpu", "host-memory", "sram-cache", "local-storage", "shared-storage", "pcie", "nic", "scale-out-switch", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "orchestration", "collectives", "inference-gateway", "inference-scheduler", "model-state", "activations", "gradients", "optimizer-state", "kv-cache" ], "notes": "CPU-only execution uses CPU arithmetic and host memory without a separate accelerator. PCIe attaches storage or network devices; collective and scale-out links apply to distributed CPU jobs. Server power/cooling and advanced chiplet packaging are deployment or product variants. Logical training and attention-cache allocations are selected by workload." }, { "architectureId": "discrete-gpu", "componentIds": [ "cpu", "accelerator", "device-memory", "host-memory", "sram-cache", "local-storage", "shared-storage", "pcie", "nic", "dpu", "scale-up-switch", "scale-out-switch", "optical-transceiver", "storage-fabric", "ups", "pdu", "psu", "cold-plate", "cdu", "heat-rejection", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "orchestration", "collectives", "inference-gateway", "inference-scheduler", "model-state", "activations", "gradients", "optimizer-state", "kv-cache", "bmc" ], "notes": "The accelerator is a GPU; device memory may be HBM or GDDR. Scale-up fabric, infrastructure DPU, optics and liquid cooling are optional. Cluster networking and facility equipment appear only at the corresponding system scale. Air and liquid paths may be alternatives or complement each other." }, { "architectureId": "integrated-soc", "componentIds": [ "cpu", "accelerator", "host-memory", "device-memory", "sram-cache", "local-storage", "edge-io", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "inference-scheduler", "model-state", "activations", "gradients", "optimizer-state", "kv-cache" ], "notes": "The accelerator represents integrated GPU and/or NPU engines. Host-memory and device-memory roles may refer to one shared physical pool and must not be drawn as duplicate RAM. Training allocations refer to supported CPU/GPU execution, not an assumption of NPU training. Edge I/O, cooling and power arrangements vary between mobile SoCs and data-center APUs." }, { "architectureId": "tensor-asic", "componentIds": [ "cpu", "accelerator", "device-memory", "host-memory", "sram-cache", "shared-storage", "nic", "scale-up-switch", "scale-out-switch", "optical-transceiver", "storage-fabric", "pdu", "psu", "cold-plate", "cdu", "heat-rejection", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "orchestration", "collectives", "inference-gateway", "inference-scheduler", "model-state", "activations", "gradients", "optimizer-state", "kv-cache" ], "notes": "The accelerator contains matrix arrays plus other engines. Scale-up-switch denotes the vendor's local accelerator fabric, including direct links; it does not imply NVSwitch or even a packet switch. TPU ICI and AWS NeuronLink remain distinct implementations. Device and cooling details vary by generation; training state is shown only for supported training configurations." }, { "architectureId": "reconfigurable-dataflow", "componentIds": [ "cpu", "accelerator", "device-memory", "host-memory", "sram-cache", "local-storage", "shared-storage", "nic", "scale-up-switch", "scale-out-switch", "pdu", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "orchestration", "collectives", "inference-gateway", "inference-scheduler", "model-state", "activations", "gradients", "optimizer-state", "kv-cache" ], "notes": "The accelerator represents the programmable compute/memory grid. Device-memory covers accelerator-attached HBM and DDR; host-memory is the separate host working-memory role. On-chip SRAM and inter-RDU links must retain their dataflow-specific behavior. Training links describe supported products such as the cited SN40L architecture, not every current serving product." }, { "architectureId": "wafer-scale", "componentIds": [ "cpu", "accelerator", "host-memory", "sram-cache", "local-storage", "shared-storage", "nic", "scale-up-switch", "scale-out-switch", "storage-fabric", "pdu", "psu", "cold-plate", "cdu", "heat-rejection", "package", "foundry", "memory-supply", "compiler-runtime", "orchestration", "collectives", "inference-gateway", "inference-scheduler", "model-state", "activations", "gradients", "optimizer-state", "kv-cache", "bmc" ], "notes": "The accelerator is the wafer-scale processor, with distributed SRAM rather than a mandatory HBM device-memory block. Package represents physical integration, not a conventional GPU interposer. Generic host/storage and network nodes can introduce external weight-streaming infrastructure, but MemoryX and SwarmX require their own family-specific detail. Scale-up-switch is a conceptual communication join, not evidence of a standard switch implementation. Cooling and training/inference arrangements depend on the selected system." }, { "architectureId": "inference-asic", "componentIds": [ "cpu", "accelerator", "device-memory", "host-memory", "sram-cache", "local-storage", "shared-storage", "pcie", "nic", "scale-up-switch", "scale-out-switch", "pdu", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "orchestration", "collectives", "inference-gateway", "inference-scheduler", "model-state", "activations", "kv-cache" ], "notes": "The accelerator is the inference-specialized engine. Device-memory applies to external-memory designs such as Inferentia2; an SRAM-centric variant can omit it. Local fabric and collectives apply when models span devices. This representative serving join omits optimizer/gradient allocations without claiming that all underlying chips are incapable of training." }, { "architectureId": "fpga-adaptive", "componentIds": [ "cpu", "accelerator", "device-memory", "host-memory", "sram-cache", "local-storage", "pcie", "nic", "edge-io", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "model-state", "activations" ], "notes": "The accelerator represents programmable logic/DSP pipelines and optional hard AI Engines; those blocks remain distinct in the expanded view. CPU can be embedded or external. External DRAM/HBM, PCIe and network interfaces are product-dependent. The join depicts a specialized inference pipeline; custom training and attention-cache designs require additional workload-specific links." }, { "architectureId": "digital-near-memory", "componentIds": [ "cpu", "accelerator", "host-memory", "device-memory", "sram-cache", "local-storage", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "model-state", "activations" ], "notes": "The accelerator represents digital processors near memory. Device-memory can represent PIM DRAM, while sram-cache represents the local SRAM of a near-compute variant; these are alternative organizations. An UPMEM DPU is a data processing unit inside DRAM and must not be joined to the catalog's dpu infrastructure/network-offload component. Selected training offloads do not establish support for a complete generic training pipeline." }, { "architectureId": "analog-in-memory", "componentIds": [ "cpu", "accelerator", "host-memory", "sram-cache", "local-storage", "psu", "package", "foundry", "compiler-runtime", "model-state", "activations" ], "notes": "The accelerator represents crossbars plus analog/digital peripheral circuits. Model-state resides as conductances in the compute array; do not invent a separate HBM chip for those weights. SRAM refers only to digital buffers/control where present. The catalog's DRAM/HBM memory-supply node does not describe PCM/ReRAM fabrication, so that specific supply-chain connection needs a future component. Digital preparation/training elsewhere is not on-device training." }, { "architectureId": "photonic-compute", "componentIds": [ "cpu", "accelerator", "device-memory", "host-memory", "sram-cache", "local-storage", "pcie", "psu", "air-cooling", "package", "foundry", "memory-supply", "compiler-runtime", "model-state", "activations" ], "notes": "The accelerator represents photonic arithmetic and its electronic control/conversion circuitry. Memory links denote electronic storage, not light-based persistent memory. PCIe and external device memory are system variants. No optical-transceiver link is inferred from photonic arithmetic: network optics are a separate optional transport layer. Foundry is a coarse manufacturing join; photonic fabrication and light-source suppliers need finer future nodes." }, { "architectureId": "neuromorphic", "componentIds": [ "cpu", "accelerator", "host-memory", "sram-cache", "local-storage", "edge-io", "psu", "package", "foundry", "compiler-runtime", "model-state", "activations" ], "notes": "The accelerator represents event-processing neuron cores and supported learning engines. Model-state represents synaptic parameters; activations is a coarse join for intermediate/state values and must expand into event and neuron-state concepts. SRAM holds local state close to compute. Event-routing networks are not automatically Ethernet, GPU scale-up fabrics or collective libraries. Standard optimizer, gradient and attention-KV allocations are not implied by local learning." } ] } }