{ "dataset": "/data/on-prem/snapshot.json", "datasetSha256": "6771b2cd66cb24f8604f97f423f6dba033da045e3d44c7e22179b035b3454b2c", "width": 1920, "figures": [ { "id": "batching", "variant": "full", "date": "2026-09-02", "title": "Slow each user from 200 to 30 tokens a second and the same GPU serves 6.7 times the tokens", "alt": "9 points, two models, one B200 One NVIDIA B200 serving GLM-5.3, 744B parameters on one eight-GPU server, and Kimi K3, 2.8T parameters across two. Each point is the best configuration InferenceX measured at that per-user speed on its AgentX workload, replayed Claude Code sessions with a median of 142k input and 444 output tokens a request. GLM-5.3’s sweep stopped at 16 clients, so its slower rungs are unmeasured. The price is the cost per million tokens at SemiAnalysis’s estimate of what a large cloud pays to own the GPU, $1.73 an hour.", "png": "batching-2026-09.png", "svg": "batching-2026-09.svg", "height": 3000, "legend": [ [ "GLM-5.3, 100 tokens a second per user", "10,968 tokens a second per GPU · $0.04 per million tokens" ], [ "GLM-5.3, 150 tokens a second per user", "8,054 tokens a second per GPU · $0.06 per million tokens" ], [ "GLM-5.3, 200 tokens a second per user", "6,809 tokens a second per GPU · $0.07 per million tokens" ], [ "Kimi K3, 30 tokens a second per user", "6,783 tokens a second per GPU · $0.07 per million tokens" ], [ "Kimi K3, 50 tokens a second per user", "4,561 tokens a second per GPU · $0.11 per million tokens" ], [ "Kimi K3, 75 tokens a second per user", "3,022 tokens a second per GPU · $0.16 per million tokens" ], [ "Kimi K3, 100 tokens a second per user", "2,147 tokens a second per GPU · $0.22 per million tokens" ], [ "Kimi K3, 150 tokens a second per user", "1,366 tokens a second per GPU · $0.35 per million tokens" ], [ "Kimi K3, 200 tokens a second per user", "1,007 tokens a second per GPU · $0.48 per million tokens" ] ] }, { "id": "batching", "variant": "chart", "date": "2026-09-02", "title": "Slow each user from 200 to 30 tokens a second and the same GPU serves 6.7 times the tokens", "alt": "9 points, two models, one B200 One NVIDIA B200 serving GLM-5.3, 744B parameters on one eight-GPU server, and Kimi K3, 2.8T parameters across two. Each point is the best configuration InferenceX measured at that per-user speed on its AgentX workload, replayed Claude Code sessions with a median of 142k input and 444 output tokens a request. GLM-5.3’s sweep stopped at 16 clients, so its slower rungs are unmeasured. The price is the cost per million tokens at SemiAnalysis’s estimate of what a large cloud pays to own the GPU, $1.73 an hour.", "png": "batching-2026-09-chart.png", "svg": "batching-2026-09-chart.svg", "height": 1800, "legend": [ [ "GLM-5.3, 100 tokens a second per user", "10,968 tokens a second per GPU · $0.04 per million tokens" ], [ "GLM-5.3, 150 tokens a second per user", "8,054 tokens a second per GPU · $0.06 per million tokens" ], [ "GLM-5.3, 200 tokens a second per user", "6,809 tokens a second per GPU · $0.07 per million tokens" ], [ "Kimi K3, 30 tokens a second per user", "6,783 tokens a second per GPU · $0.07 per million tokens" ], [ "Kimi K3, 50 tokens a second per user", "4,561 tokens a second per GPU · $0.11 per million tokens" ], [ "Kimi K3, 75 tokens a second per user", "3,022 tokens a second per GPU · $0.16 per million tokens" ], [ "Kimi K3, 100 tokens a second per user", "2,147 tokens a second per GPU · $0.22 per million tokens" ], [ "Kimi K3, 150 tokens a second per user", "1,366 tokens a second per GPU · $0.35 per million tokens" ], [ "Kimi K3, 200 tokens a second per user", "1,007 tokens a second per GPU · $0.48 per million tokens" ] ] }, { "id": "prices", "variant": "full", "date": "2026-09-08", "title": "Same B200, five prices from $1.73 to $12.36 an hour", "alt": "5 prices for one B200 hour The same NVIDIA B200, priced by who owns it. Rental rows are on-demand or capacity-block list prices, retrieved 8 September 2026. Owned rows are costs: a listed eight-GPU server written off over three years plus power at the US industrial rate, and SemiAnalysis’s estimate at hyperscaler scale. AWS prints $12.355.", "png": "prices-2026-09.png", "svg": "prices-2026-09.svg", "height": 2520, "legend": [ [ "AWS capacity block", "$12.36 per GPU hour · list price to rent · p6-b200.48xlarge, US East, $98.84 an hour for eight" ], [ "RunPod on-demand, secure cloud", "$6.79 per GPU hour · list price to rent · Community cloud is $5.98" ], [ "Lambda on-demand", "$6.69 per GPU hour · list price to rent · Eight-GPU instance" ], [ "Owned, small buyer, server and power only", "$2.07 per GPU hour · cost of owning · Derived below: the listed server over 36 months plus power at the US industrial rate. No colocation, staff, spares or networking." ], [ "Owned at hyperscaler volume, SemiAnalysis estimate", "$1.73 per GPU hour · cost of owning · SemiAnalysis AI Cloud TCO model, 'Owning at Large Hyperscaler Volume'" ] ] }, { "id": "prices", "variant": "chart", "date": "2026-09-08", "title": "Same B200, five prices from $1.73 to $12.36 an hour", "alt": "5 prices for one B200 hour The same NVIDIA B200, priced by who owns it. Rental rows are on-demand or capacity-block list prices, retrieved 8 September 2026. Owned rows are costs: a listed eight-GPU server written off over three years plus power at the US industrial rate, and SemiAnalysis’s estimate at hyperscaler scale. AWS prints $12.355.", "png": "prices-2026-09-chart.png", "svg": "prices-2026-09-chart.svg", "height": 1800, "legend": [ [ "AWS capacity block", "$12.36 per GPU hour · list price to rent · p6-b200.48xlarge, US East, $98.84 an hour for eight" ], [ "RunPod on-demand, secure cloud", "$6.79 per GPU hour · list price to rent · Community cloud is $5.98" ], [ "Lambda on-demand", "$6.69 per GPU hour · list price to rent · Eight-GPU instance" ], [ "Owned, small buyer, server and power only", "$2.07 per GPU hour · cost of owning · Derived below: the listed server over 36 months plus power at the US industrial rate. No colocation, staff, spares or networking." ], [ "Owned at hyperscaler volume, SemiAnalysis estimate", "$1.73 per GPU hour · cost of owning · SemiAnalysis AI Cloud TCO model, 'Owning at Large Hyperscaler Volume'" ] ] }, { "id": "latency", "variant": "full", "date": "2026-04-07", "title": "The slowest 1% of API calls wait 1.7 to 9.9 times the median", "alt": "10 models, 24 hours of pings Time to first token, median to 99th percentile. ModelStats’ 24 hours of pings every ten minutes from one US datacenter, April 2026, on the models it tracked then. The ratio is the 99th percentile over the median. The tick is the single slowest request in the window. This is the shape of the tail, not a current ranking.", "png": "latency-2026-09.png", "svg": "latency-2026-09.svg", "height": 3276, "legend": [ [ "Gemini 2.5 Flash Lite", "median 311 ms · 99th percentile 3.1 s · slowest 3.3 s · 9.9x the median at the 99th percentile" ], [ "Gemini 2.5 Flash", "median 477 ms · 99th percentile 1.3 s · slowest 1.9 s · 2.7x the median at the 99th percentile" ], [ "GPT-4o Mini", "median 536 ms · 99th percentile 4.5 s · slowest 5.9 s · 8.3x the median at the 99th percentile" ], [ "Claude Haiku 4.5", "median 541 ms · 99th percentile 4.1 s · slowest 11.6 s · 7.6x the median at the 99th percentile" ], [ "GPT-4o", "median 639 ms · 99th percentile 4.5 s · slowest 7.0 s · 7.1x the median at the 99th percentile" ], [ "GPT-4.1 Mini", "median 661 ms · 99th percentile 2.2 s · slowest 4.3 s · 3.3x the median at the 99th percentile" ], [ "GPT-4.1", "median 726 ms · 99th percentile 1.9 s · slowest 4.8 s · 2.7x the median at the 99th percentile" ], [ "Claude Sonnet 4.6", "median 941 ms · 99th percentile 4.8 s · slowest 5.5 s · 5.1x the median at the 99th percentile" ], [ "DeepSeek V3", "median 1.5 s · 99th percentile 2.5 s · slowest 3.1 s · 1.7x the median at the 99th percentile" ], [ "Claude Opus 4.6", "median 1.6 s · 99th percentile 3.8 s · slowest 5.3 s · 2.3x the median at the 99th percentile" ] ] }, { "id": "latency", "variant": "chart", "date": "2026-04-07", "title": "The slowest 1% of API calls wait 1.7 to 9.9 times the median", "alt": "10 models, 24 hours of pings Time to first token, median to 99th percentile. ModelStats’ 24 hours of pings every ten minutes from one US datacenter, April 2026, on the models it tracked then. The ratio is the 99th percentile over the median. The tick is the single slowest request in the window. This is the shape of the tail, not a current ranking.", "png": "latency-2026-09-chart.png", "svg": "latency-2026-09-chart.svg", "height": 1956, "legend": [ [ "Gemini 2.5 Flash Lite", "median 311 ms · 99th percentile 3.1 s · slowest 3.3 s · 9.9x the median at the 99th percentile" ], [ "Gemini 2.5 Flash", "median 477 ms · 99th percentile 1.3 s · slowest 1.9 s · 2.7x the median at the 99th percentile" ], [ "GPT-4o Mini", "median 536 ms · 99th percentile 4.5 s · slowest 5.9 s · 8.3x the median at the 99th percentile" ], [ "Claude Haiku 4.5", "median 541 ms · 99th percentile 4.1 s · slowest 11.6 s · 7.6x the median at the 99th percentile" ], [ "GPT-4o", "median 639 ms · 99th percentile 4.5 s · slowest 7.0 s · 7.1x the median at the 99th percentile" ], [ "GPT-4.1 Mini", "median 661 ms · 99th percentile 2.2 s · slowest 4.3 s · 3.3x the median at the 99th percentile" ], [ "GPT-4.1", "median 726 ms · 99th percentile 1.9 s · slowest 4.8 s · 2.7x the median at the 99th percentile" ], [ "Claude Sonnet 4.6", "median 941 ms · 99th percentile 4.8 s · slowest 5.5 s · 5.1x the median at the 99th percentile" ], [ "DeepSeek V3", "median 1.5 s · 99th percentile 2.5 s · slowest 3.1 s · 1.7x the median at the 99th percentile" ], [ "Claude Opus 4.6", "median 1.6 s · 99th percentile 3.8 s · slowest 5.3 s · 2.3x the median at the 99th percentile" ] ] }, { "id": "crossover", "variant": "full", "date": "2026-09-09", "title": "Owning GLM-5.3 breaks even at 19% of one server, Kimi K3 at 77% of two", "alt": "2 crossings, one server and two Each model’s servers against its maker’s API with every input token a cache hit, which is the bill an agent loop pays. Owning is the listed server over three years plus power, nothing else, so each owned line is flat: one server for GLM-5.3, two for Kimi K3. The rising lines are the API bill for the same tokens. Utilization is the share of the servers’ capacity at 100 tokens a second per user.", "png": "crossover-2026-09.png", "svg": "crossover-2026-09.svg", "height": 2472, "legend": [ [ "1 Owning GLM-5.3, one server", "against its cached API · 19% of one server · 1.45B tokens a day · $12,077 a month" ], [ "2 Owning Kimi K3, two servers", "against its cached API · 77% of two servers · 2.3B tokens a day · $24,155 a month" ], [ "GLM-5.3 API, every input token cached", "$0.27 per million tokens · $62,951 a month at full use, 7.58B tokens a day" ], [ "Kimi K3 API, every input token cached", "$0.35 per million tokens · $31,236 a month at full use, 2.97B tokens a day" ] ] }, { "id": "crossover", "variant": "chart", "date": "2026-09-09", "title": "Owning GLM-5.3 breaks even at 19% of one server, Kimi K3 at 77% of two", "alt": "2 crossings, one server and two Each model’s servers against its maker’s API with every input token a cache hit, which is the bill an agent loop pays. Owning is the listed server over three years plus power, nothing else, so each owned line is flat: one server for GLM-5.3, two for Kimi K3. The rising lines are the API bill for the same tokens. Utilization is the share of the servers’ capacity at 100 tokens a second per user.", "png": "crossover-2026-09-chart.png", "svg": "crossover-2026-09-chart.svg", "height": 1872, "legend": [ [ "1 Owning GLM-5.3, one server", "against its cached API · 19% of one server · 1.45B tokens a day · $12,077 a month" ], [ "2 Owning Kimi K3, two servers", "against its cached API · 77% of two servers · 2.3B tokens a day · $24,155 a month" ], [ "GLM-5.3 API, every input token cached", "$0.27 per million tokens · $62,951 a month at full use, 7.58B tokens a day" ], [ "Kimi K3 API, every input token cached", "$0.35 per million tokens · $31,236 a month at full use, 2.97B tokens a day" ] ] }, { "id": "utilization", "variant": "full", "date": "2026-09-09", "title": "The owned GLM-5.3 token costs 3.8 times the cached API at 5% and breaks even at 19%", "alt": "5 utilizations, one server, GLM-5.3 What the owned GLM-5.3 token costs at the utilizations that actually occur: the server’s $12,077 a month spread over the tokens it serves. Each bar is anchored: Cast AI’s measured average across tens of thousands of clusters, the two break-even points, the half that 83% of enterprises in VentureBeat’s survey do not reach, and full use. The rule is Z.ai’s price with every input token cached.", "png": "utilization-2026-09.png", "svg": "utilization-2026-09.svg", "height": 2400, "legend": [ [ "5%, Cast AI’s measured average", "$1.05 per million tokens · 3.8x the cached API price" ], [ "19%, breaks even with the cached API", "$0.27 per million tokens · 1.0x the cached API price" ], [ "31%, breaks even with renting at Lambda", "$0.17 per million tokens · 0.6x the cached API price" ], [ "50%, the surveys’ ceiling", "$0.11 per million tokens · 0.4x the cached API price" ], [ "100%, never sleeps", "$0.05 per million tokens · 0.2x the cached API price" ] ] }, { "id": "utilization", "variant": "chart", "date": "2026-09-09", "title": "The owned GLM-5.3 token costs 3.8 times the cached API at 5% and breaks even at 19%", "alt": "5 utilizations, one server, GLM-5.3 What the owned GLM-5.3 token costs at the utilizations that actually occur: the server’s $12,077 a month spread over the tokens it serves. Each bar is anchored: Cast AI’s measured average across tens of thousands of clusters, the two break-even points, the half that 83% of enterprises in VentureBeat’s survey do not reach, and full use. The rule is Z.ai’s price with every input token cached.", "png": "utilization-2026-09-chart.png", "svg": "utilization-2026-09-chart.svg", "height": 1728, "legend": [ [ "5%, Cast AI’s measured average", "$1.05 per million tokens · 3.8x the cached API price" ], [ "19%, breaks even with the cached API", "$0.27 per million tokens · 1.0x the cached API price" ], [ "31%, breaks even with renting at Lambda", "$0.17 per million tokens · 0.6x the cached API price" ], [ "50%, the surveys’ ceiling", "$0.11 per million tokens · 0.4x the cached API price" ], [ "100%, never sleeps", "$0.05 per million tokens · 0.2x the cached API price" ] ] } ] }