{ "schema_version": 1, "updated_at": "2026-10-27T19:32:00.807394+00:00", "site_url": "https://ai-tier.com/", "full_data_url": "https://ai-tier.com/data.json", "methodology_url": "https://ai-tier.com/methodology/", "measurement_device": "Raspberry Pi 5, 16 GB RAM; CPU unless labelled AI HAT (Hailo-8 26 TOPS).", "reading_notes": [ "Null means unavailable, never zero.", "↑ means higher is better; ↓ means lower is better.", "Scores are fixed public test selections, not full official benchmark scores.", "Compare like-for-like context, precision, reasoning and hardware configurations.", "WER can exceed 100%. TTS intelligibility does not measure voice naturalness.", "Values retain full precision. Tables and charts round only for display." ], "categories": { "text": { "url": "https://ai-tier.com/", "description": "Equal-weight average of IFEval, BFCL and RULER selections. Pi 5 timing and memory refer to the stated context and reasoning configuration.", "results": [ { "model": { "id": "qwen3-5-0-8b", "name": "Qwen3.5-0.8B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-0.8B", "configuration": "CPU", "result_id": "split-ec1e56b61819c4a36b43", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-5-0-8b/", "metrics": { "Capability / 100 ↑": 66.7825, "IFEVAL / 100 ↑": 46.875, "BFCL / 100 ↑": 64.58333333333333, "RULER / 100 ↑": 88.88916666666667, "Output tokens / second ↑": 15.089063, "Prompt tokens / second ↑": 93.549938, "First token · s ↓": 4.555013135000081, "Task time · s ↓": 20.89784112872217, "Peak memory · MiB ↓": 1267.078125, "Model files · MiB ↓": 507.847900390625, "Parameters · billions": 0.8 } }, { "model": { "id": "qwen3-5-4b", "name": "Qwen3.5-4B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-4B", "configuration": "CPU", "result_id": "split-e8760fac2b5f15ac9d03", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-5-4b/", "metrics": { "Capability / 100 ↑": 75.63659722222222, "IFEVAL / 100 ↑": 85.9375, "BFCL / 100 ↑": 75.0, "RULER / 100 ↑": 65.97229166666666, "Output tokens / second ↑": 3.361275, "Prompt tokens / second ↑": 15.566687, "First token · s ↓": 22.51384568300091, "Task time · s ↓": 94.84019198311115, "Peak memory · MiB ↓": 5690.765625, "Model files · MiB ↓": 2613.9620666503906, "Parameters · billions": 4 } }, { "model": { "id": "qwen3-5-2b", "name": "Qwen3.5-2B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-2B", "configuration": "CPU", "result_id": "split-bcfdc0c187722e2a4c09", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-5-2b/", "metrics": { "Capability / 100 ↑": 53.240763888888885, "IFEVAL / 100 ↑": 62.5, "BFCL / 100 ↑": 31.25, "RULER / 100 ↑": 65.97229166666666, "Output tokens / second ↑": 7.666992, "Prompt tokens / second ↑": 47.52552, "First token · s ↓": 8.961591903501358, "Task time · s ↓": 41.747595989388884, "Peak memory · MiB ↓": 2712.703125, "Model files · MiB ↓": 1221.500244140625, "Parameters · billions": 2 } }, { "model": { "id": "nemotron-3-nano-4b", "name": "Nemotron 3 Nano 4B", "publisher": "NVIDIA", "source_url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16", "configuration": "CPU", "result_id": "split-ef45721757e77b7ad8a9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/nemotron-3-nano-4b/", "metrics": { "Capability / 100 ↑": 77.71993055555555, "IFEVAL / 100 ↑": 67.1875, "BFCL / 100 ↑": 68.75, "RULER / 100 ↑": 97.22229166666666, "Output tokens / second ↑": 3.482796, "Prompt tokens / second ↑": 9.247137, "First token · s ↓": 44.01260239250041, "Task time · s ↓": 167.8948213242221, "Peak memory · MiB ↓": 4173.75, "Model files · MiB ↓": 2765.937530517578, "Parameters · billions": 3.97 } }, { "model": { "id": "minicpm5-2b", "name": "MiniCPM5 2B", "publisher": "OpenBMB / ModelBest", "source_url": "https://huggingface.co/openbmb/MiniCPM5-2B", "configuration": "CPU", "result_id": "split-cc594b7c5a91e8f294a3", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/minicpm5-2b/", "metrics": { "Capability / 100 ↑": 71.06493055555556, "IFEVAL / 100 ↑": 81.25, "BFCL / 100 ↑": 68.75, "RULER / 100 ↑": 63.19479166666667, "Output tokens / second ↑": 8.05251, "Prompt tokens / second ↑": 40.358482, "First token · s ↓": 10.420626442000867, "Task time · s ↓": 52.4709822935551, "Peak memory · MiB ↓": 2900.9375, "Model files · MiB ↓": 1488.9892272949219, "Parameters · billions": 2.51675648 } }, { "model": { "id": "minicpm5-1b", "name": "MiniCPM5 1B", "publisher": "OpenBMB / ModelBest", "source_url": "https://huggingface.co/openbmb/MiniCPM5-1B", "configuration": "CPU", "result_id": "split-8e3c34a7a1c5944e70af", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/minicpm5-1b/", "metrics": { "Capability / 100 ↑": 68.9236111111111, "IFEVAL / 100 ↑": 60.9375, "BFCL / 100 ↑": 77.08333333333333, "RULER / 100 ↑": 68.75, "Output tokens / second ↑": 20.474609, "Prompt tokens / second ↑": 114.750154, "First token · s ↓": 3.7106312949999847, "Task time · s ↓": 22.552227694388826, "Peak memory · MiB ↓": 1260.609375, "Model files · MiB ↓": 656.1907958984375, "Parameters · billions": 1.080632832 } }, { "model": { "id": "gemma-4-e4b-it", "name": "Gemma 4 E4B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E4B-it", "configuration": "CPU", "result_id": "split-d24ceee912504ce9dbc2", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/gemma-4-e4b-it/", "metrics": { "Capability / 100 ↑": 89.93055555555556, "IFEVAL / 100 ↑": 87.5, "BFCL / 100 ↑": 82.29166666666667, "RULER / 100 ↑": 100.0, "Output tokens / second ↑": 3.494973, "Prompt tokens / second ↑": 14.799022, "First token · s ↓": 30.18775824849945, "Task time · s ↓": 120.11333086338931, "Peak memory · MiB ↓": 7432.953125, "Model files · MiB ↓": 4746.6007080078125, "Parameters · billions": 8.0 } }, { "model": { "id": "lfm2-5-1-2b-instruct", "name": "LFM2.5-1.2B-Instruct", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-1.2B-Instruct", "configuration": "CPU", "result_id": "split-a3f3a0ff3185d62260d5", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-5-1-2b-instruct/", "metrics": { "Capability / 100 ↑": 64.61819444444444, "IFEVAL / 100 ↑": 79.6875, "BFCL / 100 ↑": 41.666666666666664, "RULER / 100 ↑": 72.50041666666667, "Output tokens / second ↑": 16.046622, "Prompt tokens / second ↑": 81.155932, "First token · s ↓": 5.3215523324997775, "Task time · s ↓": 21.918217982499602, "Peak memory · MiB ↓": 1547.1875, "Model files · MiB ↓": 697.0359497070312, "Parameters · billions": 1.17 } }, { "model": { "id": "granite-4-1-3b", "name": "Granite 4.1 3B", "publisher": "IBM", "source_url": "https://huggingface.co/ibm-granite/granite-4.1-3b", "configuration": "CPU", "result_id": "split-0955b3d9d0aeb98dbe34", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/granite-4-1-3b/", "metrics": { "Capability / 100 ↑": 82.90527777777778, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 82.29166666666667, "RULER / 100 ↑": 89.86166666666666, "Output tokens / second ↑": 4.836334, "Prompt tokens / second ↑": 24.969193, "First token · s ↓": 15.982009349498185, "Task time · s ↓": 81.82567661972159, "Peak memory · MiB ↓": 4440.75, "Model files · MiB ↓": 2002.2408142089844, "Parameters · billions": 3.40283648 } }, { "model": { "id": "qwen3-5-9b", "name": "Qwen3.5-9B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-9B", "configuration": "CPU", "result_id": "split-d2efefd0c60393ae98eb", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-5-9b/", "metrics": { "Capability / 100 ↑": 87.84722222222223, "IFEVAL / 100 ↑": 90.625, "BFCL / 100 ↑": 79.16666666666667, "RULER / 100 ↑": 93.75, "Output tokens / second ↑": 1.876366, "Prompt tokens / second ↑": 9.805878, "First token · s ↓": 42.29176578049919, "Task time · s ↓": 173.64571075377776, "Peak memory · MiB ↓": 10331.46875, "Model files · MiB ↓": 5417.368377685547, "Parameters · billions": 9 } }, { "model": { "id": "gemma-4-e2b-it", "name": "Gemma 4 E2B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E2B-it", "configuration": "CPU", "result_id": "split-a21f19657ff1bc780124", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/gemma-4-e2b-it/", "metrics": { "Capability / 100 ↑": 75.28923611111111, "IFEVAL / 100 ↑": 79.6875, "BFCL / 100 ↑": 84.375, "RULER / 100 ↑": 61.80520833333333, "Output tokens / second ↑": 7.363575, "Prompt tokens / second ↑": 31.880922, "First token · s ↓": 14.60180834699895, "Task time · s ↓": 62.077807984610544, "Peak memory · MiB ↓": 4535.1875, "Model files · MiB ↓": 2962.8164978027344, "Parameters · billions": 5.1 } }, { "model": { "id": "granite-4-2-3b", "name": "Granite 4.2 3B", "publisher": "IBM", "source_url": "https://huggingface.co/ibm-granite/granite-4.2-3b", "configuration": "CPU", "result_id": "split-6867ed377897660beb6a", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/granite-4-2-3b/", "metrics": { "Capability / 100 ↑": 59.78020833333333, "IFEVAL / 100 ↑": 67.1875, "BFCL / 100 ↑": 48.958333333333336, "RULER / 100 ↑": 63.19479166666667, "Output tokens / second ↑": 4.907236, "Prompt tokens / second ↑": 24.236827, "First token · s ↓": 18.0705806320002, "Task time · s ↓": 98.64402532333285, "Peak memory · MiB ↓": 4350.5625, "Model files · MiB ↓": 2140.056182861328, "Parameters · billions": 3.6597376 } }, { "model": { "id": "smollm3-3b", "name": "SmolLM3 3B", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolLM3-3B", "configuration": "CPU", "result_id": "split-d9da8cf0ffcffc2b5acf", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/smollm3-3b/", "metrics": { "Capability / 100 ↑": 64.87277777777778, "IFEVAL / 100 ↑": 64.0625, "BFCL / 100 ↑": 66.66666666666667, "RULER / 100 ↑": 63.88916666666667, "Output tokens / second ↑": 6.051597, "Prompt tokens / second ↑": 29.106509, "First token · s ↓": 15.144708934509254, "Task time · s ↓": 69.99642298722272, "Peak memory · MiB ↓": 4087.609375, "Model files · MiB ↓": 1826.5774841308594, "Parameters · billions": 3.0 } }, { "model": { "id": "qwen3-4b-instruct-2507", "name": "Qwen3-4B-Instruct-2507", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507", "configuration": "CPU", "result_id": "split-cb2fffe41f8676067f6a", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-4b-instruct-2507/", "metrics": { "Capability / 100 ↑": 79.91909722222222, "IFEVAL / 100 ↑": 85.9375, "BFCL / 100 ↑": 84.375, "RULER / 100 ↑": 69.44479166666666, "Output tokens / second ↑": 4.317374, "Prompt tokens / second ↑": 21.006106, "First token · s ↓": 18.51519297499908, "Task time · s ↓": 98.92973175238916, "Peak memory · MiB ↓": 5470.0625, "Model files · MiB ↓": 2381.5928649902344, "Parameters · billions": 4 } }, { "model": { "id": "ministral-3-3b-instruct-2512", "name": "Ministral 3 3B Instruct", "publisher": "Mistral AI", "source_url": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512", "configuration": "CPU", "result_id": "split-1030ded09d7ed4b7508c", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/ministral-3-3b-instruct-2512/", "metrics": { "Capability / 100 ↑": 50.57875000000001, "IFEVAL / 100 ↑": 56.25, "BFCL / 100 ↑": 84.375, "RULER / 100 ↑": 11.11125, "Output tokens / second ↑": 5.461245, "Prompt tokens / second ↑": 26.900451, "First token · s ↓": 19.1819107620031, "Task time · s ↓": null, "Peak memory · MiB ↓": 4637.875, "Model files · MiB ↓": 2047.5606994628906, "Parameters · billions": 3.849090048 } }, { "model": { "id": "granite-4-2-8b", "name": "Granite 4.2 8B", "publisher": "IBM", "source_url": "https://huggingface.co/ibm-granite/granite-4.2-8b", "configuration": "CPU", "result_id": "split-51a9360a0279278f59de", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/granite-4-2-8b/", "metrics": { "Capability / 100 ↑": 52.951458333333335, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 21.875, "RULER / 100 ↑": 60.416875, "Output tokens / second ↑": 2.023068, "Prompt tokens / second ↑": 9.184655, "First token · s ↓": 47.04735665100088, "Task time · s ↓": 257.7212806548883, "Peak memory · MiB ↓": 10368.015625, "Model files · MiB ↓": 5100.1719970703125, "Parameters · billions": 8.79159296 } }, { "model": { "id": "lfm2-5-8b-a1b", "name": "LFM2.5-8B-A1B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "configuration": "CPU", "result_id": "split-0f06b08c57cfeff42fd0", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "on" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-5-8b-a1b/", "metrics": { "Capability / 100 ↑": 34.72222222222222, "IFEVAL / 100 ↑": 71.875, "BFCL / 100 ↑": 30.208333333333332, "RULER / 100 ↑": 2.0833333333333335, "Output tokens / second ↑": 9.440216, "Prompt tokens / second ↑": 35.442923, "First token · s ↓": 10.784428085004038, "Task time · s ↓": 52.197660326276996, "Peak memory · MiB ↓": 10002.671875, "Model files · MiB ↓": 4916.729705810547, "Parameters · billions": 8.3 } }, { "model": { "id": "olmo-3-7b-instruct", "name": "Olmo 3 7B Instruct", "publisher": "Allen Institute for AI", "source_url": "https://huggingface.co/allenai/Olmo-3-7B-Instruct", "configuration": "CPU", "result_id": "split-5bb05ed4d56cedcf7600", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/olmo-3-7b-instruct/", "metrics": { "Capability / 100 ↑": 72.8586111111111, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 86.45833333333333, "RULER / 100 ↑": 55.555, "Output tokens / second ↑": 2.481016, "Prompt tokens / second ↑": 11.840787, "First token · s ↓": 34.23801488750178, "Task time · s ↓": 164.11492233538672, "Peak memory · MiB ↓": 12719.890625, "Model files · MiB ↓": 4264.4520263671875, "Parameters · billions": 7.298011136 } }, { "model": { "id": "granite-4-1-8b", "name": "Granite 4.1 8B", "publisher": "IBM", "source_url": "https://huggingface.co/ibm-granite/granite-4.1-8b", "configuration": "CPU", "result_id": "split-ff6b67eff005c35c493b", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/granite-4-1-8b/", "metrics": { "Capability / 100 ↑": 78.12506944444443, "IFEVAL / 100 ↑": 84.375, "BFCL / 100 ↑": 85.41666666666667, "RULER / 100 ↑": 64.58354166666666, "Output tokens / second ↑": 2.090342, "Prompt tokens / second ↑": 9.286639, "First token · s ↓": 41.56725073599955, "Task time · s ↓": 200.67483961749895, "Peak memory · MiB ↓": 10197.390625, "Model files · MiB ↓": 5100.168609619141, "Parameters · billions": 8.79159296 } }, { "model": { "id": "g9v3-3b", "name": "G9v3 3B", "publisher": "AI9Stars", "source_url": "https://huggingface.co/ai9stars/G9v3-3B", "configuration": "CPU", "result_id": "split-2addc77d3802d70b958e", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/g9v3-3b/", "metrics": { "Capability / 100 ↑": 66.42361111111111, "IFEVAL / 100 ↑": 67.1875, "BFCL / 100 ↑": 75.0, "RULER / 100 ↑": 57.083333333333336, "Output tokens / second ↑": 6.6879, "Prompt tokens / second ↑": 31.927707, "First token · s ↓": 13.093676286996924, "Task time · s ↓": 68.43435264666671, "Peak memory · MiB ↓": 3592.578125, "Model files · MiB ↓": 1814.4517211914062, "Parameters · billions": 2.98865664 } }, { "model": { "id": "ministral-3-8b-instruct-2512", "name": "Ministral 3 8B Instruct", "publisher": "Mistral AI", "source_url": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512", "configuration": "CPU", "result_id": "split-647974999383d619bcd8", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/ministral-3-8b-instruct-2512/", "metrics": { "Capability / 100 ↑": 38.71527777777778, "IFEVAL / 100 ↑": 60.9375, "BFCL / 100 ↑": 42.708333333333336, "RULER / 100 ↑": 12.5, "Output tokens / second ↑": 2.14964, "Prompt tokens / second ↑": 9.75835, "First token · s ↓": 49.40163753950037, "Task time · s ↓": null, "Peak memory · MiB ↓": 9830.828125, "Model files · MiB ↓": 4958.068756103516, "Parameters · billions": 8.91802624 } }, { "model": { "id": "ling-3-0-tiny", "name": "Ling 3.0 Tiny", "publisher": "inclusionAI / Ant Group", "source_url": "https://huggingface.co/inclusionAI/Ling-3.0-tiny", "configuration": "CPU", "result_id": "split-ff0d8284411f1d27de21", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/ling-3-0-tiny/", "metrics": { "Capability / 100 ↑": 83.101875, "IFEVAL / 100 ↑": 65.625, "BFCL / 100 ↑": 84.375, "RULER / 100 ↑": 99.305625, "Output tokens / second ↑": 10.109197, "Prompt tokens / second ↑": 51.808093, "First token · s ↓": 8.839614408501802, "Task time · s ↓": 44.946184082611985, "Peak memory · MiB ↓": 9245.703125, "Model files · MiB ↓": 4689.554870605469, "Parameters · billions": 7.8933928 } }, { "model": { "id": "ornith-1-5-9b", "name": "Ornith 1.5 9B", "publisher": "Ornith AI", "source_url": "https://huggingface.co/ornith-ai/Ornith-1.5-9B", "configuration": "CPU", "result_id": "split-f90f0ea9763c8e85d75d", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/ornith-1-5-9b/", "metrics": { "Capability / 100 ↑": 57.986111111111114, "IFEVAL / 100 ↑": 65.625, "BFCL / 100 ↑": 35.416666666666664, "RULER / 100 ↑": 72.91666666666667, "Output tokens / second ↑": 2.001515, "Prompt tokens / second ↑": 10.007523, "First token · s ↓": 43.03577033200054, "Task time · s ↓": 194.05502745811222, "Peak memory · MiB ↓": 10794.828125, "Model files · MiB ↓": 5636.9617919921875, "Parameters · billions": 9.653104368 } }, { "model": { "id": "qwen3-0-6b", "name": "Qwen3-0.6B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-0.6B", "configuration": "CPU", "result_id": "split-3c9e27a9d7bb30531273", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-0-6b/", "metrics": { "Capability / 100 ↑": 71.81736111111111, "IFEVAL / 100 ↑": 51.5625, "BFCL / 100 ↑": 72.91666666666667, "RULER / 100 ↑": 90.97291666666666, "Output tokens / second ↑": 28.381631, "Prompt tokens / second ↑": 154.964081, "First token · s ↓": 2.7853344135000953, "Task time · s ↓": 21.722934131944285, "Peak memory · MiB ↓": 1296.75, "Model files · MiB ↓": 378.32781982421875, "Parameters · billions": 0.6 } }, { "model": { "id": "phi-4-mini-instruct", "name": "Phi-4 Mini Instruct", "publisher": "Microsoft", "source_url": "https://huggingface.co/microsoft/Phi-4-mini-instruct", "configuration": "CPU", "result_id": "split-c026e7ea0589af0d4602", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/phi-4-mini-instruct/", "metrics": { "Capability / 100 ↑": 79.88430555555556, "IFEVAL / 100 ↑": 71.875, "BFCL / 100 ↑": 75.0, "RULER / 100 ↑": 92.77791666666667, "Output tokens / second ↑": 4.758097, "Prompt tokens / second ↑": 23.849997, "First token · s ↓": 16.749959718003083, "Task time · s ↓": 86.44235150250097, "Peak memory · MiB ↓": 5386.703125, "Model files · MiB ↓": 2376.436492919922, "Parameters · billions": 3.8 } }, { "model": { "id": "qwen3-1-7b", "name": "Qwen3-1.7B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-1.7B", "configuration": "CPU", "result_id": "split-cb25f4000397189445dc", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-1-7b/", "metrics": { "Capability / 100 ↑": 68.98159722222222, "IFEVAL / 100 ↑": 65.625, "BFCL / 100 ↑": 84.375, "RULER / 100 ↑": 56.94479166666667, "Output tokens / second ↑": 10.377891, "Prompt tokens / second ↑": 56.919873, "First token · s ↓": 7.220681736998813, "Task time · s ↓": 41.95774999499958, "Peak memory · MiB ↓": 2596.09375, "Model files · MiB ↓": 1223.0293884277344, "Parameters · billions": 1.7 } }, { "model": { "id": "qwen3-8b", "name": "Qwen3-8B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-8B", "configuration": "CPU", "result_id": "split-9600f3bc996d00177a09", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-8b/", "metrics": { "Capability / 100 ↑": 76.67833333333333, "IFEVAL / 100 ↑": 79.6875, "BFCL / 100 ↑": 84.375, "RULER / 100 ↑": 65.9725, "Output tokens / second ↑": 2.371461, "Prompt tokens / second ↑": 10.708131, "First token · s ↓": 36.840145805006614, "Task time · s ↓": 170.7631036106112, "Peak memory · MiB ↓": 9454.40625, "Model files · MiB ↓": 4794.867980957031, "Parameters · billions": 8 } }, { "model": { "id": "qwen3-4b", "name": "Qwen3-4B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-4B", "configuration": "CPU", "result_id": "split-0abe1418dc701bf5fc16", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-4b/", "metrics": { "Capability / 100 ↑": 86.80555555555556, "IFEVAL / 100 ↑": 78.125, "BFCL / 100 ↑": 82.29166666666667, "RULER / 100 ↑": 100.0, "Output tokens / second ↑": 4.729427, "Prompt tokens / second ↑": 22.171823, "First token · s ↓": 18.60446619600407, "Task time · s ↓": 101.52690705627901, "Peak memory · MiB ↓": 5470.515625, "Model files · MiB ↓": 2381.592041015625, "Parameters · billions": 4 } }, { "model": { "id": "smollm2-1-7b-instruct", "name": "SmolLM2 1.7B Instruct", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct", "configuration": "CPU", "result_id": "split-2da546a1366138e6258a", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/smollm2-1-7b-instruct/", "metrics": { "Capability / 100 ↑": 47.30319444444444, "IFEVAL / 100 ↑": 45.3125, "BFCL / 100 ↑": 34.375, "RULER / 100 ↑": 62.22208333333333, "Output tokens / second ↑": 10.757987, "Prompt tokens / second ↑": 50.593392, "First token · s ↓": 8.69361437549378, "Task time · s ↓": 46.809230910997336, "Peak memory · MiB ↓": 2890.65625, "Model files · MiB ↓": 1006.7077026367188, "Parameters · billions": 1.7 } }, { "model": { "id": "hermes-3-llama-3-1-8b", "name": "Hermes 3 Llama 3.1 8B", "publisher": "Nous Research", "source_url": "https://huggingface.co/NousResearch/Hermes-3-Llama-3.1-8B", "configuration": "CPU", "result_id": "split-30468f73a575097c7221", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/hermes-3-llama-3-1-8b/", "metrics": { "Capability / 100 ↑": 75.17368055555556, "IFEVAL / 100 ↑": 57.8125, "BFCL / 100 ↑": 69.79166666666667, "RULER / 100 ↑": 97.916875, "Output tokens / second ↑": 2.572344, "Prompt tokens / second ↑": 11.023624, "First token · s ↓": 37.3712729195031, "Task time · s ↓": 164.03173546322392, "Peak memory · MiB ↓": 9780.6875, "Model files · MiB ↓": 4692.778228759766, "Parameters · billions": 8.0 } }, { "model": { "id": "olmo-2-0425-1b-instruct", "name": "OLMo 2 1B Instruct", "publisher": "Allen Institute for AI", "source_url": "https://huggingface.co/allenai/OLMo-2-0425-1B-Instruct", "configuration": "CPU", "result_id": "split-c61692dc39a077092b62", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/olmo-2-0425-1b-instruct/", "metrics": { "Capability / 100 ↑": 43.51847222222222, "IFEVAL / 100 ↑": 62.5, "BFCL / 100 ↑": 20.833333333333332, "RULER / 100 ↑": 47.22208333333333, "Output tokens / second ↑": 14.086423, "Prompt tokens / second ↑": 77.105112, "First token · s ↓": 5.268431024996971, "Task time · s ↓": 27.349324330944608, "Peak memory · MiB ↓": 2144.578125, "Model files · MiB ↓": 892.1769104003906, "Parameters · billions": 1.484916736 } }, { "model": { "id": "lfm2-700m", "name": "LFM2-700M", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2-700M", "configuration": "CPU", "result_id": "split-7b3cf958313cc2114b36", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-700m/", "metrics": { "Capability / 100 ↑": 67.22229166666666, "IFEVAL / 100 ↑": 62.5, "BFCL / 100 ↑": 79.16666666666667, "RULER / 100 ↑": 60.00020833333333, "Output tokens / second ↑": 24.720899, "Prompt tokens / second ↑": 130.662968, "First token · s ↓": 3.305278237501625, "Task time · s ↓": 14.80111590271958, "Peak memory · MiB ↓": 1039.171875, "Model files · MiB ↓": 446.91497802734375, "Parameters · billions": 0.7 } }, { "model": { "id": "lfm2-1-2b", "name": "LFM2-1.2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2-1.2B", "configuration": "CPU", "result_id": "split-5018295a8bbfa1cafab9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-1-2b/", "metrics": { "Capability / 100 ↑": 59.50236111111111, "IFEVAL / 100 ↑": 60.9375, "BFCL / 100 ↑": 73.95833333333333, "RULER / 100 ↑": 43.61125, "Output tokens / second ↑": 16.012313, "Prompt tokens / second ↑": 80.171915, "First token · s ↓": 5.342279178999888, "Task time · s ↓": 24.005661617722783, "Peak memory · MiB ↓": 1547.25, "Model files · MiB ↓": 697.0339050292969, "Parameters · billions": 1.17 } }, { "model": { "id": "tulu-3-1-llama-3-1-8b", "name": "Tulu 3.1 Llama 3.1 8B", "publisher": "Allen Institute for AI", "source_url": "https://huggingface.co/allenai/Llama-3.1-Tulu-3.1-8B", "configuration": "CPU", "result_id": "split-6c80743b243b24d791e9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/tulu-3-1-llama-3-1-8b/", "metrics": { "Capability / 100 ↑": 79.08569444444444, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 62.5, "RULER / 100 ↑": 98.19458333333333, "Output tokens / second ↑": 2.594058, "Prompt tokens / second ↑": 10.983182, "First token · s ↓": 37.08278820949636, "Task time · s ↓": 162.408417674556, "Peak memory · MiB ↓": 9347.8125, "Model files · MiB ↓": 4692.8226318359375, "Parameters · billions": 8.0 } }, { "model": { "id": "lfm2-8b-a1b", "name": "LFM2-8B-A1B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2-8B-A1B", "configuration": "CPU", "result_id": "split-4ab707e886a841ea6156", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-8b-a1b/", "metrics": { "Capability / 100 ↑": 70.61347222222223, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 54.166666666666664, "RULER / 100 ↑": 81.11125, "Output tokens / second ↑": 11.686644, "Prompt tokens / second ↑": 38.172127, "First token · s ↓": 10.948165878995496, "Task time · s ↓": 42.0463042297213, "Peak memory · MiB ↓": 9762.984375, "Model files · MiB ↓": 4811.076843261719, "Parameters · billions": 8.3 } }, { "model": { "id": "lfm2-5-230m", "name": "LFM2.5-230M", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-230M", "configuration": "CPU", "result_id": "split-9cee2abae5e07d7885e9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-5-230m/", "metrics": { "Capability / 100 ↑": 46.52777777777778, "IFEVAL / 100 ↑": 56.25, "BFCL / 100 ↑": 33.333333333333336, "RULER / 100 ↑": 50.0, "Output tokens / second ↑": 72.257778, "Prompt tokens / second ↑": 451.030828, "First token · s ↓": 0.9612192685017362, "Task time · s ↓": 5.014768272221166, "Peak memory · MiB ↓": 407.78125, "Model files · MiB ↓": 146.29965209960938, "Parameters · billions": 0.23 } }, { "model": { "id": "lfm2-5-350m", "name": "LFM2.5-350M", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-350M", "configuration": "CPU", "result_id": "split-45fe625d405a65acf0e0", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-5-350m/", "metrics": { "Capability / 100 ↑": 47.12979166666667, "IFEVAL / 100 ↑": 71.875, "BFCL / 100 ↑": 23.958333333333332, "RULER / 100 ↑": 45.556041666666665, "Output tokens / second ↑": 50.907912, "Prompt tokens / second ↑": 266.845925, "First token · s ↓": 1.5653364379977575, "Task time · s ↓": 7.054372154610367, "Peak memory · MiB ↓": 566.828125, "Model files · MiB ↓": 218.68917846679688, "Parameters · billions": 0.35 } }, { "model": { "id": "smollm2-135m-instruct", "name": "SmolLM2 135M Instruct", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolLM2-135M-Instruct", "configuration": "CPU", "result_id": "split-20c7c0ad2f1a510d8e98", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/smollm2-135m-instruct/", "metrics": { "Capability / 100 ↑": 26.50451388888889, "IFEVAL / 100 ↑": 15.625, "BFCL / 100 ↑": 22.916666666666668, "RULER / 100 ↑": 40.971875, "Output tokens / second ↑": 97.616351, "Prompt tokens / second ↑": 265.093541, "First token · s ↓": 1.6852133054999285, "Task time · s ↓": 10.212339876279454, "Peak memory · MiB ↓": 298.609375, "Model files · MiB ↓": 100.56890869140625, "Parameters · billions": 0.135 } }, { "model": { "id": "smollm2-360m-instruct", "name": "SmolLM2 360M Instruct", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolLM2-360M-Instruct", "configuration": "CPU", "result_id": "split-f4904df75beed6925a4f", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/smollm2-360m-instruct/", "metrics": { "Capability / 100 ↑": 30.358819444444446, "IFEVAL / 100 ↑": 23.4375, "BFCL / 100 ↑": 31.25, "RULER / 100 ↑": 36.388958333333335, "Output tokens / second ↑": 42.104354, "Prompt tokens / second ↑": 98.614487, "First token · s ↓": 4.441433799504011, "Task time · s ↓": 23.332233069835638, "Peak memory · MiB ↓": 593.15625, "Model files · MiB ↓": 258.0552673339844, "Parameters · billions": 0.36 } }, { "model": { "id": "lfm2-350m", "name": "LFM2-350M", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2-350M", "configuration": "CPU", "result_id": "split-e3071e86d9c840211d5a", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm2-350m/", "metrics": { "Capability / 100 ↑": 47.326388888888886, "IFEVAL / 100 ↑": 42.1875, "BFCL / 100 ↑": 65.625, "RULER / 100 ↑": 34.166666666666664, "Output tokens / second ↑": 49.520648, "Prompt tokens / second ↑": 273.788292, "First token · s ↓": 1.5834343609894859, "Task time · s ↓": 7.65854995149938, "Peak memory · MiB ↓": 566.96875, "Model files · MiB ↓": 218.68624877929688, "Parameters · billions": 0.35 } }, { "model": { "id": "qwen3-vl-4b-instruct", "name": "Qwen3-VL 4B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct", "configuration": "CPU", "result_id": "split-bd337818738fd2f20845", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-vl-4b-instruct/", "metrics": { "Capability / 100 ↑": 88.59958333333333, "IFEVAL / 100 ↑": 79.6875, "BFCL / 100 ↑": 87.5, "RULER / 100 ↑": 98.61125, "Output tokens / second ↑": 4.763826, "Prompt tokens / second ↑": 22.230791, "First token · s ↓": 18.409301656502066, "Task time · s ↓": 94.63502254649858, "Peak memory · MiB ↓": 5488.515625, "Model files · MiB ↓": 2381.5933837890625, "Parameters · billions": 4.437815808 } }, { "model": { "id": "qwen3-vl-8b-instruct", "name": "Qwen3-VL 8B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct", "configuration": "CPU", "result_id": "split-dec48ffd7d596479219e", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen3-vl-8b-instruct/", "metrics": { "Capability / 100 ↑": 90.1042361111111, "IFEVAL / 100 ↑": 85.9375, "BFCL / 100 ↑": 86.45833333333333, "RULER / 100 ↑": 97.916875, "Output tokens / second ↑": 2.428279, "Prompt tokens / second ↑": 10.850894, "First token · s ↓": 36.10395761249674, "Task time · s ↓": 164.46981041166225, "Peak memory · MiB ↓": 9493.59375, "Model files · MiB ↓": 4794.869232177734, "Parameters · billions": 8.767123696 } }, { "model": { "id": "spark-x2-5-1-7b", "name": "Spark-X2.5 1.7B", "publisher": "SparkLLM", "source_url": "https://huggingface.co/XHToken/Spark-X2.5-1.7B", "configuration": "CPU", "result_id": "split-1cbcc365a42fae56a211", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/spark-x2-5-1-7b/", "metrics": { "Capability / 100 ↑": 65.08111111111111, "IFEVAL / 100 ↑": 67.1875, "BFCL / 100 ↑": 35.416666666666664, "RULER / 100 ↑": 92.63916666666667, "Output tokens / second ↑": 10.460564, "Prompt tokens / second ↑": 55.188589, "First token · s ↓": 8.36503645349876, "Task time · s ↓": 34.53024214405418, "Peak memory · MiB ↓": 2377.921875, "Model files · MiB ↓": 1056.1541137695312, "Parameters · billions": 1.707657216 } }, { "model": { "id": "spark-x2-5-4b", "name": "Spark-X2.5 4B", "publisher": "SparkLLM", "source_url": "https://huggingface.co/XHToken/Spark-X2.5-4B", "configuration": "CPU", "result_id": "split-4107c1a3341dd754490b", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/spark-x2-5-4b/", "metrics": { "Capability / 100 ↑": 62.09493055555556, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 45.833333333333336, "RULER / 100 ↑": 63.888958333333335, "Output tokens / second ↑": 4.393894, "Prompt tokens / second ↑": 20.956846, "First token · s ↓": 22.197481900511775, "Task time · s ↓": 97.01186719527566, "Peak memory · MiB ↓": 5461.78125, "Model files · MiB ↓": 2479.7671813964844, "Parameters · billions": 4.11207936 } }, { "model": { "id": "nanbeige4-2-3b", "name": "Nanbeige4.2 3B", "publisher": "Nanbeige", "source_url": "https://huggingface.co/Nanbeige/Nanbeige4.2-3B", "configuration": "CPU", "result_id": "split-9533d35fc868a20e76e9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/nanbeige4-2-3b/", "metrics": { "Capability / 100 ↑": 62.50006944444444, "IFEVAL / 100 ↑": 75.0, "BFCL / 100 ↑": 47.916666666666664, "RULER / 100 ↑": 64.58354166666666, "Output tokens / second ↑": 2.677974, "Prompt tokens / second ↑": 12.169813, "First token · s ↓": 39.077668161000474, "Task time · s ↓": 218.4109528041671, "Peak memory · MiB ↓": 5301.640625, "Model files · MiB ↓": 2559.6847229003906, "Parameters · billions": 4.169800704 } }, { "model": { "id": "agents-a1-4b", "name": "Agents-A1 4B", "publisher": "InternScience", "source_url": "https://huggingface.co/InternScience/Agents-A1-4B", "configuration": "CPU", "result_id": "split-493a9d762e5bbc03ad00", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/agents-a1-4b/", "metrics": { "Capability / 100 ↑": 62.326388888888886, "IFEVAL / 100 ↑": 79.6875, "BFCL / 100 ↑": 78.125, "RULER / 100 ↑": 29.166666666666668, "Output tokens / second ↑": 3.508441, "Prompt tokens / second ↑": 19.324029, "First token · s ↓": 19.382119542511646, "Task time · s ↓": null, "Peak memory · MiB ↓": 5612.4375, "Model files · MiB ↓": 2583.3180541992188, "Parameters · billions": 4.539265536 } }, { "model": { "id": "llama-3-2-1b-instruct", "name": "Llama 3.2 1B Instruct", "publisher": "Meta", "source_url": "https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct", "configuration": "CPU", "result_id": "split-90ee888c189b78ae49b9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/llama-3-2-1b-instruct/", "metrics": { "Capability / 100 ↑": 50.74069444444444, "IFEVAL / 100 ↑": 40.625, "BFCL / 100 ↑": 44.791666666666664, "RULER / 100 ↑": 66.80541666666667, "Output tokens / second ↑": 14.482261, "Prompt tokens / second ↑": 84.989714, "First token · s ↓": 5.011380100506358, "Task time · s ↓": 27.94589233206029, "Peak memory · MiB ↓": 1796.046875, "Model files · MiB ↓": 770.2773742675781, "Parameters · billions": 1.2358144 } }, { "model": { "id": "llama-3-2-3b-instruct", "name": "Llama 3.2 3B Instruct", "publisher": "Meta", "source_url": "https://huggingface.co/meta-llama/Llama-3.2-3B-Instruct", "configuration": "CPU", "result_id": "split-8a67380bb030e73d511a", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/llama-3-2-3b-instruct/", "metrics": { "Capability / 100 ↑": 77.09493055555556, "IFEVAL / 100 ↑": 70.3125, "BFCL / 100 ↑": 72.91666666666667, "RULER / 100 ↑": 88.055625, "Output tokens / second ↑": 5.943614, "Prompt tokens / second ↑": 28.913852, "First token · s ↓": 14.766089211509097, "Task time · s ↓": 70.40388091883344, "Peak memory · MiB ↓": 4432.875, "Model files · MiB ↓": 1925.8285522460938, "Parameters · billions": 3.212749824 } }, { "model": { "id": "llama-3-1-8b-instruct", "name": "Llama 3.1 8B Instruct", "publisher": "Meta", "source_url": "https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct", "configuration": "CPU", "result_id": "split-e46b227f0bc9f081bfa9", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/llama-3-1-8b-instruct/", "metrics": { "Capability / 100 ↑": 75.11576388888888, "IFEVAL / 100 ↑": 78.125, "BFCL / 100 ↑": 81.25, "RULER / 100 ↑": 65.97229166666666, "Output tokens / second ↑": 2.566538, "Prompt tokens / second ↑": 10.958147, "First token · s ↓": 38.923492285990505, "Task time · s ↓": 165.74162761644402, "Peak memory · MiB ↓": 9808.625, "Model files · MiB ↓": 4692.782623291016, "Parameters · billions": 8.030261248 } }, { "model": { "id": "qwen2-5-0-5b-instruct", "name": "Qwen2.5 0.5B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct", "configuration": "CPU", "result_id": "split-268e210256756f89886a", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen2-5-0-5b-instruct/", "metrics": { "Capability / 100 ↑": 50.05798611111111, "IFEVAL / 100 ↑": 26.5625, "BFCL / 100 ↑": 58.333333333333336, "RULER / 100 ↑": 65.278125, "Output tokens / second ↑": 29.114445, "Prompt tokens / second ↑": 91.470047, "First token · s ↓": 4.375849988486152, "Task time · s ↓": 20.511573052720955, "Peak memory · MiB ↓": 744.953125, "Model files · MiB ↓": 468.6355895996094, "Parameters · billions": 0.494032768 } }, { "model": { "id": "qwen2-5-1-5b-instruct", "name": "Qwen2.5 1.5B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct", "configuration": "CPU", "result_id": "split-14360d841ba3c882fefd", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen2-5-1-5b-instruct/", "metrics": { "Capability / 100 ↑": 71.58590277777778, "IFEVAL / 100 ↑": 42.1875, "BFCL / 100 ↑": 82.29166666666667, "RULER / 100 ↑": 90.27854166666667, "Output tokens / second ↑": 11.938406, "Prompt tokens / second ↑": 61.426436, "First token · s ↓": 6.588526071514934, "Task time · s ↓": 32.714297831781245, "Peak memory · MiB ↓": 2069.578125, "Model files · MiB ↓": 1065.5600891113281, "Parameters · billions": 1.543714304 } }, { "model": { "id": "qwen2-5-3b-instruct", "name": "Qwen2.5 3B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-3B-Instruct", "configuration": "CPU", "result_id": "split-d9148c9543203511b6cf", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen2-5-3b-instruct/", "metrics": { "Capability / 100 ↑": 76.21541666666667, "IFEVAL / 100 ↑": 57.8125, "BFCL / 100 ↑": 79.16666666666667, "RULER / 100 ↑": 91.66708333333334, "Output tokens / second ↑": 5.956884, "Prompt tokens / second ↑": 29.168904, "First token · s ↓": 13.831023981008912, "Task time · s ↓": 66.49912269449851, "Peak memory · MiB ↓": 3909.21875, "Model files · MiB ↓": 2007.4203186035156, "Parameters · billions": 3.085938688 } }, { "model": { "id": "qwen2-5-7b-instruct", "name": "Qwen2.5 7B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", "configuration": "CPU", "result_id": "split-114d367931b0ea5a10ec", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/qwen2-5-7b-instruct/", "metrics": { "Capability / 100 ↑": 85.82194444444444, "IFEVAL / 100 ↑": 73.4375, "BFCL / 100 ↑": 89.58333333333333, "RULER / 100 ↑": 94.445, "Output tokens / second ↑": 2.721918, "Prompt tokens / second ↑": 11.557519, "First token · s ↓": 35.207831467996584, "Task time · s ↓": 154.99202376649677, "Peak memory · MiB ↓": 8947.515625, "Model files · MiB ↓": 4466.127044677734, "Parameters · billions": 7.615616512 } }, { "model": { "id": "spark-x2-6-4b", "name": "Spark-X2.6 4B", "publisher": "SparkLLM", "source_url": "https://huggingface.co/XHToken/Spark-X2.6-4B", "configuration": "CPU", "result_id": "split-989419d142d5f91cdfe6", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/spark-x2-6-4b/", "metrics": { "Capability / 100 ↑": 82.9861111111111, "IFEVAL / 100 ↑": 71.875, "BFCL / 100 ↑": 77.08333333333333, "RULER / 100 ↑": 100.0, "Output tokens / second ↑": 2.50438, "Prompt tokens / second ↑": 19.176016, "First token · s ↓": 23.82938736648066, "Task time · s ↓": 114.02966959349254, "Peak memory · MiB ↓": 5938.0, "Model files · MiB ↓": 3039.2889099121094, "Parameters · billions": 4.11207936 } }, { "model": { "id": "lfm3-1-7b", "name": "LFM3-1.7B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-1.7B", "configuration": "CPU", "result_id": "split-f6e21a62c909b8d1244e", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm3-1-7b/", "metrics": { "Capability / 100 ↑": 48.09027777777778, "IFEVAL / 100 ↑": 76.5625, "BFCL / 100 ↑": 34.375, "RULER / 100 ↑": 33.333333333333336, "Output tokens / second ↑": 10.308286, "Prompt tokens / second ↑": 55.796265, "First token · s ↓": 7.368729968497064, "Task time · s ↓": 33.65913388488762, "Peak memory · MiB ↓": 2285.484375, "Model files · MiB ↓": 1048.6867980957031, "Parameters · billions": 1.728801536 } }, { "model": { "id": "lfm3-vl-2b", "name": "LFM3-VL-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-VL-2B", "configuration": "CPU", "result_id": "split-6d201ec8a1d32fc1b434", "context_tokens": 4096, "quantization": "Q4_K_M", "reasoning_mode": "off" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/lfm3-vl-2b/", "metrics": { "Capability / 100 ↑": 63.865833333333335, "IFEVAL / 100 ↑": 68.75, "BFCL / 100 ↑": 30.208333333333332, "RULER / 100 ↑": 92.63916666666667, "Output tokens / second ↑": 10.313649, "Prompt tokens / second ↑": 56.208774, "First token · s ↓": 7.363880627439357, "Task time · s ↓": 30.881954527078456, "Peak memory · MiB ↓": 2285.4375, "Model files · MiB ↓": 1410.1255798339844, "Parameters · billions": 1.918181888 } }, { "model": { "id": "k2-horizon-7b", "name": "K2 Horizon 7B", "publisher": "Institute of Foundation Models / MBZUAI", "source_url": "https://huggingface.co/IFM/K2-Horizon-7B", "configuration": "CPU", "result_id": "split-57e0fd923630540d8cf5", "context_tokens": 40960, "quantization": "Q4_K_M", "reasoning_mode": "on" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/k2-horizon-7b/", "metrics": { "Capability / 100 ↑": 60.06944444444445, "IFEVAL / 100 ↑": 75.0, "BFCL / 100 ↑": 36.458333333333336, "RULER / 100 ↑": 68.75, "Output tokens / second ↑": 2.193354, "Prompt tokens / second ↑": 10.395809, "First token · s ↓": 36.14048147451831, "Task time · s ↓": 229.5505582472704, "Peak memory · MiB ↓": 14498.3125, "Model files · MiB ↓": 5333.1546630859375, "Parameters · billions": 8.99917824 } }, { "model": { "id": "k2-horizon-0-9b", "name": "K2 Horizon 0.9B", "publisher": "Institute of Foundation Models / MBZUAI", "source_url": "https://huggingface.co/IFM/K2-Horizon-0.9B", "configuration": "CPU", "result_id": "split-0d544bc5de3ffe448170", "context_tokens": 40960, "quantization": "Q4_K_M", "reasoning_mode": "on" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/k2-horizon-0-9b/", "metrics": { "Capability / 100 ↑": 53.87729166666667, "IFEVAL / 100 ↑": 67.1875, "BFCL / 100 ↑": 52.083333333333336, "RULER / 100 ↑": 42.361041666666665, "Output tokens / second ↑": 17.810669, "Prompt tokens / second ↑": 86.11311, "First token · s ↓": 5.052223982522264, "Task time · s ↓": 42.30955060595685, "Peak memory · MiB ↓": 3469.78125, "Model files · MiB ↓": 635.3226318359375, "Parameters · billions": 1.078285824 } }, { "model": { "id": "k2-horizon-3-7b", "name": "K2 Horizon 3.7B", "publisher": "Institute of Foundation Models / MBZUAI", "source_url": "https://huggingface.co/IFM/K2-Horizon-3.7B", "configuration": "CPU", "result_id": "split-5a57437c64a9770de8bd", "context_tokens": 40960, "quantization": "Q4_K_M", "reasoning_mode": "on" }, "result_url": "https://ai-tier.com/results/raspberry-pi-5/k2-horizon-3-7b/", "metrics": { "Capability / 100 ↑": 65.10416666666667, "IFEVAL / 100 ↑": 73.4375, "BFCL / 100 ↑": 57.291666666666664, "RULER / 100 ↑": 64.58333333333333, "Output tokens / second ↑": 3.911827, "Prompt tokens / second ↑": 21.12, "First token · s ↓": 19.15180052400683, "Task time · s ↓": 144.8433887452799, "Peak memory · MiB ↓": 10355.921875, "Model files · MiB ↓": 3010.3665771484375, "Parameters · billions": 5.05825536 } } ] }, "vision": { "url": "https://ai-tier.com/vision", "columns": [ "VQAv2 / 100 ↑", "POPE / 100 ↑", "Pi 5 first token · s ↓", "Pi 5 task time · s ↓", "Peak memory · MiB ↓" ], "description": "VQAv2 tests answers to image questions. POPE checks whether named objects are present. Each uses 96 fixed questions; higher is better. Speed and memory are measured on Pi 5, including the image encoder.", "results": [ { "model": { "id": "qwen3-5-0-8b", "name": "Qwen3.5-0.8B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-0.8B", "configuration": "CPU", "result_id": "88ee173b2c548508127d", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 79.6875, "POPE / 100 ↑": 90.625, "Pi 5 first token · s ↓": 4.07251732100849, "Pi 5 task time · s ↓": 4.290834783296709, "Peak memory · MiB ↓": 1590.5625 } }, { "model": { "id": "qwen3-5-2b", "name": "Qwen3.5-2B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-2B", "configuration": "CPU", "result_id": "a696fa2c5ae3cacf36c3", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 84.58333333333333, "POPE / 100 ↑": 92.70833333333333, "Pi 5 first token · s ↓": 10.759122761999606, "Pi 5 task time · s ↓": 11.172503135187542, "Peak memory · MiB ↓": 3498.203125 } }, { "model": { "id": "qwen3-5-4b", "name": "Qwen3.5-4B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-4B", "configuration": "CPU", "result_id": "50bbccfaaa63df59398c", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 88.4375, "POPE / 100 ↑": 87.5, "Pi 5 first token · s ↓": 17.399057932008873, "Pi 5 task time · s ↓": 18.218457877445704, "Peak memory · MiB ↓": 6526.03125 } }, { "model": { "id": "qwen3-5-9b", "name": "Qwen3.5-9B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-9B", "configuration": "CPU", "result_id": "60d7422345851ba2a92e", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 86.14583333333333, "POPE / 100 ↑": 90.625, "Pi 5 first token · s ↓": 29.76885630900506, "Pi 5 task time · s ↓": 30.686355111593944, "Peak memory · MiB ↓": 11050.546875 } }, { "model": { "id": "gemma-4-e2b-it", "name": "Gemma 4 E2B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E2B-it", "configuration": "CPU", "result_id": "7c29b016c3a47ce9ea85", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 54.0625, "POPE / 100 ↑": 68.75, "Pi 5 first token · s ↓": 6.795368074992439, "Pi 5 task time · s ↓": 7.337638900110808, "Peak memory · MiB ↓": 5544.828125 } }, { "model": { "id": "gemma-4-e4b-it", "name": "Gemma 4 E4B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E4B-it", "configuration": "CPU", "result_id": "8f76223d01111f7a0e2e", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 54.166666666666664, "POPE / 100 ↑": 80.20833333333333, "Pi 5 first token · s ↓": 10.686719489007373, "Pi 5 task time · s ↓": 11.567743079370874, "Peak memory · MiB ↓": 8795.6875 } }, { "model": { "id": "ministral-3-3b-instruct-2512", "name": "Ministral 3 3B Instruct", "publisher": "Mistral AI", "source_url": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512", "configuration": "CPU", "result_id": "35f296425dbc781253c7", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 78.02083333333333, "POPE / 100 ↑": 79.16666666666667, "Pi 5 first token · s ↓": 212.86907954000344, "Pi 5 task time · s ↓": 227.65520043907412, "Peak memory · MiB ↓": 6049.359375 } }, { "model": { "id": "ministral-3-8b-instruct-2512", "name": "Ministral 3 8B Instruct", "publisher": "Mistral AI", "source_url": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512", "configuration": "CPU", "result_id": "2a8318e6145026b913c5", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 72.5, "POPE / 100 ↑": 77.08333333333333, "Pi 5 first token · s ↓": 263.6690346999967, "Pi 5 task time · s ↓": 280.3763613590746, "Peak memory · MiB ↓": 11434.515625 } }, { "model": { "id": "qwen3-vl-4b-instruct", "name": "Qwen3-VL 4B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct", "configuration": "CPU", "result_id": "480db6b7494b128d0c91", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 81.97916666666667, "POPE / 100 ↑": 90.625, "Pi 5 first token · s ↓": 16.411501037015114, "Pi 5 task time · s ↓": 17.07381551329551, "Peak memory · MiB ↓": 7169.203125 } }, { "model": { "id": "qwen3-vl-8b-instruct", "name": "Qwen3-VL 8B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct", "configuration": "CPU", "result_id": "7bde4f83afda1ad3ce5f", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 86.45833333333333, "POPE / 100 ↑": 93.75, "Pi 5 first token · s ↓": 28.563327809009934, "Pi 5 task time · s ↓": 29.60230789944331, "Peak memory · MiB ↓": 11254.0625 } }, { "model": { "id": "minicpm-v-4-7-1b", "name": "MiniCPM-V 4.7 1B", "publisher": "OpenBMB / ModelBest", "source_url": "https://huggingface.co/openbmb/MiniCPM-V-4.7-1B", "configuration": "CPU", "result_id": "bb9f67533e78c9ca0395", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 77.8125, "POPE / 100 ↑": 88.54166666666667, "Pi 5 first token · s ↓": 111.68969765197835, "Pi 5 task time · s ↓": 138.2528539901886, "Peak memory · MiB ↓": 2454.515625 } }, { "model": { "id": "lfm2-5-vl-1-6b", "name": "LFM2.5-VL-1.6B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B", "configuration": "CPU", "result_id": "714c16ea8552bb449922", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 87.60416666666667, "POPE / 100 ↑": 88.54166666666667, "Pi 5 first token · s ↓": 10.951911983982427, "Pi 5 task time · s ↓": 11.332953208295361, "Peak memory · MiB ↓": 2504.875 } }, { "model": { "id": "lfm2-5-vl-3b", "name": "LFM2.5-VL-3B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-VL-3B", "configuration": "CPU", "result_id": "f30228162c1110a3ce90", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 89.47916666666667, "POPE / 100 ↑": 92.70833333333333, "Pi 5 first token · s ↓": 14.967396296997322, "Pi 5 task time · s ↓": 15.528660613890409, "Peak memory · MiB ↓": 4369.328125 } }, { "model": { "id": "smolvlm2-256m", "name": "SmolVLM2 256M", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolVLM2-256M-Video-Instruct", "configuration": "CPU", "result_id": "933dd5027316e1dea51b", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 61.666666666666664, "POPE / 100 ↑": 68.75, "Pi 5 first token · s ↓": 6.849205981998239, "Pi 5 task time · s ↓": 6.912055933705738, "Peak memory · MiB ↓": 687.953125 } }, { "model": { "id": "smolvlm2-2-2b", "name": "SmolVLM2 2.2B", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolVLM2-2.2B-Instruct", "configuration": "CPU", "result_id": "c35d5975ad4d6fb76a09", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 79.27083333333333, "POPE / 100 ↑": 86.45833333333333, "Pi 5 first token · s ↓": 21.923474856012035, "Pi 5 task time · s ↓": 22.115156299076734, "Peak memory · MiB ↓": 4483.59375 } }, { "model": { "id": "lfm3-vl-2b", "name": "LFM3-VL-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-VL-2B", "configuration": "CPU", "result_id": "c0a30f0be74edb59d739", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 87.60416666666667, "POPE / 100 ↑": 86.45833333333333, "Pi 5 first token · s ↓": 6.760107452981174, "Pi 5 task time · s ↓": 6.981122049886768, "Peak memory · MiB ↓": 2864.890625 } }, { "model": { "id": "lfm3-omni-2b", "name": "LFM3-Omni-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-Omni-2B", "configuration": "CPU", "result_id": "1ac871d84e4c6b0081fc", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "VQAv2 / 100 ↑": 77.60416666666667, "POPE / 100 ↑": 89.58333333333333, "Pi 5 first token · s ↓": 6.71664990496356, "Pi 5 task time · s ↓": 6.9625851617117105, "Peak memory · MiB ↓": 3429.25 } } ] }, "video": { "url": "https://ai-tier.com/video", "columns": [ "Video questions · % ↑", "Event overlap · % ↑", "Pi 5 task time · s ↓", "Model files · MiB" ], "description": "MVBench uses 32 questions about short clips, shown as eight ordered frames. Event overlap tests when an action happens. Higher is better for both; these small selections do not measure live camera streaming.", "results": [ { "model": { "id": "qwen3-5-0-8b", "name": "Qwen3.5-0.8B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-0.8B", "configuration": "CPU", "result_id": "video-1b1c946187fe86f0a3a414bc", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 15.625, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 703.3389587402344 } }, { "model": { "id": "qwen3-5-2b", "name": "Qwen3.5-2B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-2B", "configuration": "CPU", "result_id": "video-efbc8d556a727d1174e90242", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 81.25, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 1858.7714233398438 } }, { "model": { "id": "qwen3-5-4b", "name": "Qwen3.5-4B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-4B", "configuration": "CPU", "result_id": "video-559ef498b125a7c00484424e", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 78.125, "Event overlap · % ↑": 21.19908089114398, "Pi 5 task time · s ↓": null, "Model files · MiB": 3255.2351989746094 } }, { "model": { "id": "qwen3-5-9b", "name": "Qwen3.5-9B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3.5-9B", "configuration": "CPU", "result_id": "video-02622879506064d25fb187f1", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 71.875, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 6292.999786376953 } }, { "model": { "id": "gemma-4-e2b-it", "name": "Gemma 4 E2B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E2B-it", "configuration": "CPU", "result_id": "video-5e20f6d22e623d2023121758", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 40.625, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 3902.8094787597656 } }, { "model": { "id": "gemma-4-e4b-it", "name": "Gemma 4 E4B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E4B-it", "configuration": "CPU", "result_id": "video-a2074bc550b083787bf46fde", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 40.625, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 5691.093688964844 } }, { "model": { "id": "ministral-3-3b-instruct-2512", "name": "Ministral 3 3B Instruct", "publisher": "Mistral AI", "source_url": "https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512", "configuration": "CPU", "result_id": "video-e25e5e6a19d73fb85082b1e2", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 59.375, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 2850.0794067382812 } }, { "model": { "id": "ministral-3-8b-instruct-2512", "name": "Ministral 3 8B Instruct", "publisher": "Mistral AI", "source_url": "https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512", "configuration": "CPU", "result_id": "video-f82cb29b34b0d4ef646dbe0f", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 37.5, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 5776.591369628906 } }, { "model": { "id": "qwen3-vl-4b-instruct", "name": "Qwen3-VL 4B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct", "configuration": "CPU", "result_id": "video-da78c3816d9dc0d4173c4616", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 71.875, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 3179.0370178222656 } }, { "model": { "id": "qwen3-vl-8b-instruct", "name": "Qwen3-VL 8B Instruct", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct", "configuration": "CPU", "result_id": "video-c8889950808dd957dac354e4", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 71.875, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 5900.206207275391 } }, { "model": { "id": "minicpm-v-4-7-1b", "name": "MiniCPM-V 4.7 1B", "publisher": "OpenBMB / ModelBest", "source_url": "https://huggingface.co/openbmb/MiniCPM-V-4.7-1B", "configuration": "CPU", "result_id": "video-cb1ab5a0278172869c2cfc46", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 78.125, "Event overlap · % ↑": 5.782016824603108, "Pi 5 task time · s ↓": null, "Model files · MiB": 1699.5636596679688 } }, { "model": { "id": "lfm2-5-vl-1-6b", "name": "LFM2.5-VL-1.6B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B", "configuration": "CPU", "result_id": "video-1af97ac329a0c7f5501ee3ff", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 65.625, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 1511.468994140625 } }, { "model": { "id": "lfm2-5-vl-3b", "name": "LFM2.5-VL-3B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-VL-3B", "configuration": "CPU", "result_id": "video-8596784e3b54524316bd37e5", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 65.625, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 2411.31689453125 } }, { "model": { "id": "smolvlm2-256m", "name": "SmolVLM2 256M", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolVLM2-256M-Video-Instruct", "configuration": "CPU", "result_id": "video-3546948171d4e4ce6070451c", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 40.625, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 306.3846740722656 } }, { "model": { "id": "smolvlm2-2-2b", "name": "SmolVLM2 2.2B", "publisher": "Hugging Face", "source_url": "https://huggingface.co/HuggingFaceTB/SmolVLM2-2.2B-Instruct", "configuration": "CPU", "result_id": "video-986d9b96b170074eab4f1703", "profile": "Eight sampled frames · Q4_K_M · 16-token answer limit", "quantization": "Q4_K_M", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 62.5, "Event overlap · % ↑": null, "Pi 5 task time · s ↓": null, "Model files · MiB": 1892.9541931152344 } }, { "model": { "id": "videojev-4b", "name": "VideoJev 4B", "publisher": "MCG-NJU", "source_url": "https://huggingface.co/ZhuYuhan/VideoJev-4B", "configuration": "CPU", "result_id": "video-eaa6f46b9613f5ac8acc1f46", "profile": "Native video decision head · BF16 · eight QA / sixteen event frames", "quantization": "BF16", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Video questions · % ↑": 71.875, "Event overlap · % ↑": 48.95027989153376, "Pi 5 task time · s ↓": null, "Model files · MiB": 8547.243767738342 } } ] }, "audio": { "url": "https://ai-tier.com/audio", "columns": [ "Clean speech WER · % ↓", "Harder speech WER · % ↓", "English sentences WER · % ↓", "German sentences WER · % ↓", "Spoken instructions · % ↑" ], "description": "Word error rate (WER) counts missed, replaced and extra words per 100 reference words; lower is better and values can exceed 100%. Spoken IFEval tests following instructions read aloud; higher is better.", "results": [ { "model": { "id": "gemma-4-e2b-it", "name": "Gemma 4 E2B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E2B-it", "configuration": "CPU", "result_id": "3128be2bc470ab16310c", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 4.403794037940379, "Harder speech WER · % ↓": 9.768339768339768, "English sentences WER · % ↓": 11.670480549199084, "German sentences WER · % ↓": 9.63740458015267, "Spoken instructions · % ↑": 55.208333333333336 } }, { "model": { "id": "gemma-4-e4b-it", "name": "Gemma 4 E4B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E4B-it", "configuration": "CPU", "result_id": "621ebe791077f8f7962f", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 4.098915989159892, "Harder speech WER · % ↓": 9.536679536679536, "English sentences WER · % ↓": 9.382151029748284, "German sentences WER · % ↓": 9.875954198473282, "Spoken instructions · % ↑": 66.66666666666666 } }, { "model": { "id": "qwen3-asr-0-6b", "name": "Qwen3-ASR 0.6B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-ASR-0.6B", "configuration": "CPU", "result_id": "4ccb02dac01c23fe90e7", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 2.6084010840108403, "Harder speech WER · % ↓": 4.324324324324325, "English sentences WER · % ↓": 5.995423340961098, "German sentences WER · % ↓": 8.826335877862595, "Spoken instructions · % ↑": null } }, { "model": { "id": "qwen3-asr-1-7b", "name": "Qwen3-ASR 1.7B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B", "configuration": "CPU", "result_id": "2ba76ec2978340fda140", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 1.7953929539295392, "Harder speech WER · % ↓": 3.3204633204633205, "English sentences WER · % ↓": 4.3478260869565215, "German sentences WER · % ↓": 5.152671755725191, "Spoken instructions · % ↑": null } }, { "model": { "id": "voxtral-mini-3b-2507", "name": "Voxtral Mini 3B", "publisher": "Mistral", "source_url": "https://huggingface.co/mistralai/Voxtral-Mini-3B-2507", "configuration": "CPU", "result_id": "a4a0265c805b7cc0e68a", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 47.89972899728997, "Harder speech WER · % ↓": 80.54054054054055, "English sentences WER · % ↓": 43.066361556064074, "German sentences WER · % ↓": 24.47519083969466, "Spoken instructions · % ↑": 31.25 } }, { "model": { "id": "lfm2-5-audio-1-5b", "name": "LFM2.5-Audio-1.5B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B", "configuration": "CPU", "result_id": "49f994cf011ed19b3231", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 2.33739837398374, "Harder speech WER · % ↓": 4.362934362934363, "English sentences WER · % ↓": 6.636155606407323, "German sentences WER · % ↓": null, "Spoken instructions · % ↑": null } }, { "model": { "id": "qwen2-5-omni-3b", "name": "Qwen2.5 Omni 3B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-Omni-3B", "configuration": "CPU", "result_id": "44145c82d2b187ff8dc8", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 2.540650406504065, "Harder speech WER · % ↓": 24.517374517374517, "English sentences WER · % ↓": 9.107551487414188, "German sentences WER · % ↓": 13.311068702290076, "Spoken instructions · % ↑": 37.5 } }, { "model": { "id": "ultravox-0-5-llama-3-2-1b", "name": "Ultravox 0.5 1B", "publisher": "Fixie", "source_url": "https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b", "configuration": "CPU", "result_id": "56a5ff768ed032ce857c", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 247.08672086720867, "Harder speech WER · % ↓": 286.0617760617761, "English sentences WER · % ↓": 220.09153318077804, "German sentences WER · % ↓": 92.60496183206106, "Spoken instructions · % ↑": 40.625 } }, { "model": { "id": "qwen2-5-omni-7b", "name": "Qwen2.5 Omni 7B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-Omni-7B", "configuration": "CPU", "result_id": "fa3e3263aba9b8c02797", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 20.426829268292682, "Harder speech WER · % ↓": 67.91505791505791, "English sentences WER · % ↓": 86.95652173913044, "German sentences WER · % ↓": 135.25763358778627, "Spoken instructions · % ↑": 26.041666666666668 } }, { "model": { "id": "lfm3-audio-2b", "name": "LFM3-Audio-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-Audio-2B", "configuration": "CPU", "result_id": "160838b52f9db6dec490", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 2.913279132791328, "Harder speech WER · % ↓": 5.521235521235521, "English sentences WER · % ↓": 6.132723112128146, "German sentences WER · % ↓": 6.6316793893129775, "Spoken instructions · % ↑": 32.29166666666667 } }, { "model": { "id": "lfm3-omni-2b", "name": "LFM3-Omni-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-Omni-2B", "configuration": "CPU", "result_id": "4881e597b77813c045d1", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Clean speech WER · % ↓": 2.845528455284553, "Harder speech WER · % ↓": 5.444015444015444, "English sentences WER · % ↓": 6.178489702517163, "German sentences WER · % ↓": 7.108778625954199, "Spoken instructions · % ↑": 65.625 } } ] }, "stt": { "url": "https://ai-tier.com/stt", "columns": [ "Overall English WER · % ↓", "Overall German WER · % ↓", "Pi 5 processing factor ↓", "Peak memory · MiB ↓" ], "description": "Overall recognition averages English word error rates from LibriSpeech clean, LibriSpeech other and FLEURS equally. German uses FLEURS alone. A Pi 5 processing factor below 1 means faster than playback.", "results": [ { "model": { "id": "whisper-tiny", "name": "Whisper Tiny", "publisher": "OpenAI", "source_url": "https://huggingface.co/openai/whisper-tiny", "configuration": "CPU", "result_id": "7aa5ca2f46fb9f607b7a", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 13.59408901003476, "Overall German WER · % ↓": 32.58587786259542, "Pi 5 processing factor ↓": 0.19145220347162142, "Peak memory · MiB ↓": 963.03125 } }, { "model": { "id": "moonshine-tiny", "name": "Moonshine Tiny", "publisher": "Moonshine AI", "source_url": "https://huggingface.co/moonshine-ai/moonshine-tiny", "configuration": "CPU", "result_id": "b67769005d290e3fddd7", "profile": null, "quantization": "INT8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 10.542423059166934, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.05374096136626905, "Peak memory · MiB ↓": 332.21875 } }, { "model": { "id": "parakeet-tdt-0-6b-v3", "name": "Parakeet TDT 0.6B v3", "publisher": "NVIDIA", "source_url": "https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3", "configuration": "CPU", "result_id": "480f397049b07932c071", "profile": null, "quantization": "INT8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 5.391754606327358, "Overall German WER · % ↓": 7.872137404580153, "Pi 5 processing factor ↓": 0.11208646026561332, "Peak memory · MiB ↓": 2271.28125 } }, { "model": { "id": "whisper-base", "name": "Whisper Base", "publisher": "OpenAI", "source_url": "https://huggingface.co/openai/whisper-base", "configuration": "CPU", "result_id": "644e6eee4409d6f9083c", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 9.638897462751903, "Overall German WER · % ↓": 19.65648854961832, "Pi 5 processing factor ↓": 0.36478092694024317, "Peak memory · MiB ↓": 1516.4375 } }, { "model": { "id": "moonshine-base", "name": "Moonshine Base", "publisher": "Moonshine AI", "source_url": "https://huggingface.co/moonshine-ai/moonshine-base", "configuration": "CPU", "result_id": "d3465b8b9aabf84901d7", "profile": null, "quantization": "INT8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 8.838396876851975, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.0970993162384985, "Peak memory · MiB ↓": 551.921875 } }, { "model": { "id": "whisper-small", "name": "Whisper Small", "publisher": "OpenAI", "source_url": "https://huggingface.co/openai/whisper-small", "configuration": "CPU", "result_id": "4030b77596ae4101834b", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 6.256761569453427, "Overall German WER · % ↓": 10.877862595419847, "Pi 5 processing factor ↓": 1.2552008673103325, "Peak memory · MiB ↓": 3766.71875 } }, { "model": { "id": "underdog-bark-3-0", "name": "Underdog Bark 3.0", "publisher": "Conway Research / Underdog", "source_url": "https://huggingface.co/ConwayResearch/Underdog-Bark-3.0", "configuration": "CPU", "result_id": "5bc6157b7c26217dcc4b", "profile": null, "quantization": "Official compressed weights, decoded to FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.895785283685044, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } }, { "model": { "id": "fluid-speech-mini", "name": "Fluid Speech Mini", "publisher": "FluidVoice", "source_url": "https://huggingface.co/FluidVoice-Labs/fluid-speech-mini-onnx", "configuration": "CPU", "result_id": "3a045e647b24692009db1254", "profile": null, "quantization": "Publisher mixed 2/4-bit ONNX; 4 threads, greedy reference decoder", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.112026018316134, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.330049356620465, "Peak memory · MiB ↓": 479.625 } }, { "model": { "id": "whisper-tiny-hailo8", "name": "Whisper Tiny", "publisher": "OpenAI", "source_url": "https://huggingface.co/openai/whisper-tiny", "configuration": "AI HAT", "result_id": "f7f17683b551159a2f051a4b", "profile": null, "quantization": "Hailo compiled INT8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 42.04538311503763, "Overall German WER · % ↓": 58.396946564885496, "Pi 5 processing factor ↓": 0.15364103922996497, "Peak memory · MiB ↓": 557.578125 } }, { "model": { "id": "whisper-base-hailo8", "name": "Whisper Base", "publisher": "OpenAI", "source_url": "https://huggingface.co/openai/whisper-base", "configuration": "AI HAT", "result_id": "e24e08573c97aee3504e50c4", "profile": null, "quantization": "Hailo compiled INT8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 28.506580616448343, "Overall German WER · % ↓": 54.48473282442748, "Pi 5 processing factor ↓": 0.2927869056164068, "Peak memory · MiB ↓": 608.375 } }, { "model": { "id": "gemma-4-e2b-it", "name": "Gemma 4 E2B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E2B-it", "configuration": "CPU", "result_id": "3128be2bc470ab16310c", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 8.614204785159744, "Overall German WER · % ↓": 9.63740458015267, "Pi 5 processing factor ↓": 1.5909146282797444, "Peak memory · MiB ↓": 5654.421875 } }, { "model": { "id": "gemma-4-e4b-it", "name": "Gemma 4 E4B IT", "publisher": "Google DeepMind", "source_url": "https://huggingface.co/google/gemma-4-E4B-it", "configuration": "CPU", "result_id": "621ebe791077f8f7962f", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 7.672582185195904, "Overall German WER · % ↓": 9.875954198473282, "Pi 5 processing factor ↓": 3.1836484544940884, "Peak memory · MiB ↓": 8230.546875 } }, { "model": { "id": "qwen3-asr-0-6b", "name": "Qwen3-ASR 0.6B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-ASR-0.6B", "configuration": "CPU", "result_id": "4ccb02dac01c23fe90e7", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.309382916432088, "Overall German WER · % ↓": 8.826335877862595, "Pi 5 processing factor ↓": 0.49297213753457775, "Peak memory · MiB ↓": 2600.640625 } }, { "model": { "id": "qwen3-asr-1-7b", "name": "Qwen3-ASR 1.7B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen3-ASR-1.7B", "configuration": "CPU", "result_id": "2ba76ec2978340fda140", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.1545607871164605, "Overall German WER · % ↓": 5.152671755725191, "Pi 5 processing factor ↓": 1.216636834864515, "Peak memory · MiB ↓": 5016.046875 } }, { "model": { "id": "voxtral-mini-3b-2507", "name": "Voxtral Mini 3B", "publisher": "Mistral", "source_url": "https://huggingface.co/mistralai/Voxtral-Mini-3B-2507", "configuration": "CPU", "result_id": "a4a0265c805b7cc0e68a", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 57.168877031298194, "Overall German WER · % ↓": 24.47519083969466, "Pi 5 processing factor ↓": 15.629256813340177, "Peak memory · MiB ↓": 6280.828125 } }, { "model": { "id": "lfm2-5-audio-1-5b", "name": "LFM2.5-Audio-1.5B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM2.5-Audio-1.5B", "configuration": "CPU", "result_id": "49f994cf011ed19b3231", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.445496114441808, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.7558340057058527, "Peak memory · MiB ↓": 2924.03125 } }, { "model": { "id": "qwen2-5-omni-3b", "name": "Qwen2.5 Omni 3B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-Omni-3B", "configuration": "CPU", "result_id": "44145c82d2b187ff8dc8", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 12.05519213709759, "Overall German WER · % ↓": 13.311068702290076, "Pi 5 processing factor ↓": 9.490129875887765, "Peak memory · MiB ↓": 6812.34375 } }, { "model": { "id": "ultravox-0-5-llama-3-2-1b", "name": "Ultravox 0.5 1B", "publisher": "Fixie", "source_url": "https://huggingface.co/fixie-ai/ultravox-v0_5-llama-3_2-1b", "configuration": "CPU", "result_id": "56a5ff768ed032ce857c", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 251.0800100365876, "Overall German WER · % ↓": 92.60496183206106, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": 3476.453125 } }, { "model": { "id": "qwen2-5-omni-7b", "name": "Qwen2.5 Omni 7B", "publisher": "Qwen", "source_url": "https://huggingface.co/Qwen/Qwen2.5-Omni-7B", "configuration": "CPU", "result_id": "fa3e3263aba9b8c02797", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 58.432802974160346, "Overall German WER · % ↓": 135.25763358778627, "Pi 5 processing factor ↓": 16.57987781319821, "Peak memory · MiB ↓": 11168.828125 } }, { "model": { "id": "lfm3-audio-2b", "name": "LFM3-Audio-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-Audio-2B", "configuration": "CPU", "result_id": "160838b52f9db6dec490", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.855745922051665, "Overall German WER · % ↓": 6.6316793893129775, "Pi 5 processing factor ↓": 0.7153966630053256, "Peak memory · MiB ↓": 3100.1875 } }, { "model": { "id": "lfm3-omni-2b", "name": "LFM3-Omni-2B", "publisher": "Liquid AI", "source_url": "https://huggingface.co/LiquidAI/LFM3-Omni-2B", "configuration": "CPU", "result_id": "4881e597b77813c045d1", "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.822677867272387, "Overall German WER · % ↓": 7.108778625954199, "Pi 5 processing factor ↓": 0.703846656962833, "Peak memory · MiB ↓": 3468.40625 } } ] }, "tts": { "url": "https://ai-tier.com/tts", "columns": [ "Overall English WER · % ↓", "Overall German WER · % ↓", "Pi 5 processing factor ↓", "Peak memory · MiB ↓" ], "description": "Intelligibility averages word error rates from Qwen ASR and Whisper transcribing generated speech. English and German stay separate. This does not score voice naturalness; original audio samples are available above. A Pi 5 processing factor below 1 means faster than playback.", "results": [ { "model": { "id": "kokoro-82m-v1", "name": "Kokoro 82M v1", "publisher": "Hexgrad", "source_url": "https://huggingface.co/hexgrad/Kokoro-82M", "configuration": "CPU", "result_id": "bb1f8fb5197860f127c4", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.311867525298988, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 1.0114763644061238, "Peak memory · MiB ↓": 830.203125 } }, { "model": { "id": "piper-lessac-medium", "name": "Piper Lessac Medium", "publisher": "Piper", "source_url": "https://huggingface.co/rhasspy/piper-voices", "configuration": "CPU", "result_id": "d427a5001b8c7fceae3d", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 4.8298068077276906, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.12324952965127195, "Peak memory · MiB ↓": 358.78125 } }, { "model": { "id": "piper-thorsten-medium", "name": "Piper Thorsten Medium", "publisher": "Piper", "source_url": "https://huggingface.co/rhasspy/piper-voices", "configuration": "CPU", "result_id": "370794f583ce2cbc5aa3", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": null, "Overall German WER · % ↓": 9.760956175298805, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } }, { "model": { "id": "kitten-nano-0-8", "name": "Kitten Nano 0.8", "publisher": "Kitten ML", "source_url": "https://huggingface.co/KittenML/kitten-tts-nano-0.8-fp32", "configuration": "CPU", "result_id": "f28d9164b96ea50548fd", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.2888684452621897, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.2413607721628822, "Peak memory · MiB ↓": 314.890625 } }, { "model": { "id": "kitten-micro-0-8", "name": "Kitten Micro 0.8", "publisher": "Kitten ML", "source_url": "https://huggingface.co/KittenML/kitten-tts-micro-0.8", "configuration": "CPU", "result_id": "1681b87dbaac8c720ee2", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.1738730450781967, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 0.5410405579118953, "Peak memory · MiB ↓": 333.953125 } }, { "model": { "id": "kitten-mini-0-8", "name": "Kitten Mini 0.8", "publisher": "Kitten ML", "source_url": "https://huggingface.co/KittenML/kitten-tts-mini-0.8", "configuration": "CPU", "result_id": "79140a1bec9fe05aca31", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.5418583256669733, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 1.0799397932412342, "Peak memory · MiB ↓": 509.625 } }, { "model": { "id": "ming-3-0-tts-0-5b", "name": "Ming 3.0 TTS 0.5B", "publisher": "inclusionAI / Ant Group", "source_url": "https://huggingface.co/inclusionAI/Ming-3.0-tts-0.5B", "configuration": "CPU", "result_id": "e01e414bc5520b69fc98", "profile": null, "quantization": "RTX BF16 / Pi FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.4498620055197793, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": 50.07756469499401, "Peak memory · MiB ↓": 8079.546875 } }, { "model": { "id": "supertonic-3", "name": "Supertonic 3", "publisher": "Supertone", "source_url": "https://huggingface.co/Supertone/supertonic-3", "configuration": "CPU", "result_id": "9b8ae3fcb75473a98b3b", "profile": null, "quantization": "FP32 ONNX", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.7258509659613614, "Overall German WER · % ↓": 10.831673306772908, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } }, { "model": { "id": "pocket-tts-18m-2026-10", "name": "Pocket TTS 18.2M", "publisher": "Kyutai", "source_url": "https://huggingface.co/kyutai/pocket-tts-without-voice-cloning/tree/a3055dcb533cf2ec90458008a01e7cef550bae2a/languages/english_18m_2026-10", "configuration": "CPU", "result_id": "1aa1c0756eb92344635d", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 7.405703771849126, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } }, { "model": { "id": "pocket-tts-7m-2026-10", "name": "Pocket TTS 6.9M", "publisher": "Kyutai", "source_url": "https://huggingface.co/kyutai/pocket-tts-without-voice-cloning/tree/a3055dcb533cf2ec90458008a01e7cef550bae2a/languages/english_7m_2026-10", "configuration": "CPU", "result_id": "be3b11b8e37b0ca828db", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 6.7387304507819685, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } }, { "model": { "id": "pocket-tts-2m-2026-10", "name": "Pocket TTS 1.9M", "publisher": "Kyutai", "source_url": "https://huggingface.co/kyutai/pocket-tts-without-voice-cloning/tree/a3055dcb533cf2ec90458008a01e7cef550bae2a/languages/english_2m_2026-10", "configuration": "CPU", "result_id": "4723add9fa77806d0d02", "profile": null, "quantization": "FP32", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 9.728610855565776, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } }, { "model": { "id": "phonon-31m-en", "name": "Phonon 31M English", "publisher": "Gradium", "source_url": "https://huggingface.co/gradium/phonon-31m-en", "configuration": "CPU", "result_id": "0e8a1d6d89b1e3a3dfad", "profile": null, "quantization": "Q8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Overall English WER · % ↓": 3.518859245630175, "Overall German WER · % ↓": null, "Pi 5 processing factor ↓": null, "Peak memory · MiB ↓": null } } ] }, "ocr": { "url": "https://ai-tier.com/ocr", "columns": [ "Text checks passed · % ↑", "Pi 5 seconds per page ↓", "Peak memory · MiB ↓" ], "description": "Text recovery and reading order on 32 English document pages, scored with 72 checks. The overall score weights four document categories equally. Higher is better. This is separate from table, equation and page-layout quality.", "results": [ { "model": { "id": "pp-ocrv4-mobile", "name": "PP-OCRv4 mobile", "publisher": "PaddlePaddle", "source_url": "https://github.com/PaddlePaddle/PaddleOCR", "configuration": "CPU", "result_id": null, "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 30.828373015873016, "Pi 5 seconds per page ↓": 29.509343732332734, "Peak memory · MiB ↓": 688.21875 } }, { "model": { "id": "pp-ocrv6-tiny", "name": "PP-OCRv6 tiny", "publisher": "PaddlePaddle", "source_url": "https://github.com/PaddlePaddle/PaddleOCR", "configuration": "CPU", "result_id": null, "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 31.200396825396826, "Pi 5 seconds per page ↓": 5.2887347871647865, "Peak memory · MiB ↓": 576.28125 } }, { "model": { "id": "pp-ocrv6-small", "name": "PP-OCRv6 small", "publisher": "PaddlePaddle", "source_url": "https://github.com/PaddlePaddle/PaddleOCR", "configuration": "CPU", "result_id": null, "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 32.0188492063492, "Pi 5 seconds per page ↓": 18.636470464999245, "Peak memory · MiB ↓": 774.59375 } }, { "model": { "id": "pp-ocrv6-medium", "name": "PP-OCRv6 medium", "publisher": "PaddlePaddle", "source_url": "https://github.com/PaddlePaddle/PaddleOCR", "configuration": "CPU", "result_id": null, "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 32.83730158730159, "Pi 5 seconds per page ↓": 78.71518286133586, "Peak memory · MiB ↓": 1617.15625 } }, { "model": { "id": "pp-ocrv5-mobile", "name": "PP-OCRv5 mobile", "publisher": "PaddlePaddle", "source_url": "https://github.com/PaddlePaddle/PaddleOCR", "configuration": "CPU", "result_id": null, "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 30.456349206349206, "Pi 5 seconds per page ↓": 24.886547669651918, "Peak memory · MiB ↓": 765.53125 } }, { "model": { "id": "indra-ocr-1-1", "name": "Indra OCR 1.1", "publisher": "Datalab", "source_url": "https://huggingface.co/datalab-to/indra-ocr", "configuration": "CPU", "result_id": null, "profile": null, "quantization": null, "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 59.59821428571429, "Pi 5 seconds per page ↓": 2.015963086279549, "Peak memory · MiB ↓": 710.046875 } }, { "model": { "id": "pp-ocrv5-mobile-hailo8", "name": "PP-OCRv5 mobile", "publisher": "PaddlePaddle", "source_url": "https://huggingface.co/PaddlePaddle/PP-OCRv5_mobile_rec", "configuration": "AI HAT", "result_id": null, "profile": null, "quantization": "Hailo compiled INT8", "context_tokens": null, "reasoning_mode": null }, "metrics": { "Text checks passed · % ↑": 10.714285714285714, "Pi 5 seconds per page ↓": 2.8078110622235624, "Peak memory · MiB ↓": 187.46875 } } ] } } }