docs(types): fix axiom/*.h typo causing -Wcomment #43

Workflow file for this run

.github/workflows/benchmarks.yml at 8f70d6d

	name: Benchmarks vs TensorFlow

	# Rigorous comparison suite. each job runs one category end-to-end:
	# build Axiom, install TF, run the Axiom bench, run the TF sidecar, merge
	# results with benchmarks/summarize.py, upload artifacts and print a
	# markdown table in the job summary.

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	schedule:
	- cron: '0 3 * * *'
	workflow_dispatch:

	concurrency:
	group: bench-${{ github.ref }}
	cancel-in-progress: true

	env:
	CMAKE_BUILD_TYPE: Release
	# gate for regressions — jobs fail when Axiom is >10% slower than TF on
	# any case in the joined table.
	REGRESSION_PCT: "10"

	jobs:
	# ======================================================================
	# 0. CALIBRATE — measure omp fork/join overhead, derive parallel
	# thresholds, and run the opt-in gemm tile sweep. surfaces the
	# calibrated numbers to the job summary so regressions in runtime
	# cost are visible. runs in parallel with parity; does not gate.
	# ======================================================================
	calibrate:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: show hardware
	run: lscpu \| grep -E '(Model name\|CPU$s$\|Thread\|Core\|Cache\|MHz)' \| head -10

	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_gemm_suite

	- name: capture calibration lines
	run: \|
	# default calibration (B) always runs at init; (A) opt-in.
	AX_GEMM_CALIBRATE=1 ./build/bench_gemm_suite 2>calib_stderr.log 1>/dev/null
	grep -E '^axiom:' calib_stderr.log \| tee calib.txt

	- name: baseline vs calibrated gemm
	run: \|
	# baseline: default tiles, default thresholds
	./build/bench_gemm_suite 2>&1 > gemm_default.txt
	# with gemm tile sweep enabled
	AX_GEMM_CALIBRATE=1 ./build/bench_gemm_suite 2>&1 > gemm_calibrated.txt

	- name: summarize
	run: \|
	{
	echo "# Calibration report"
	echo ""
	echo "## Measured"
	echo '```'
	cat calib.txt
	echo '```'
	echo ""
	echo "## GEMM defaults vs calibrated"
	echo ""
	echo "\| case \| default \| calibrated \|"
	echo "\|---\|---\|---\|"
	for case in nn_512x512x512 nn_1024x1024x1024 nn_2048x2048x2048 \
	nn_256x1024x2048 nn_256x4096x1024 nn_4096x4096x4096; do
	d=$(grep "^RESULT gemm $case gflops" gemm_default.txt \| awk '{printf "%.1f", $5}')
	c=$(grep "^RESULT gemm $case gflops" gemm_calibrated.txt \| awk '{printf "%.1f", $5}')
	echo "\| $case \| $d GFLOPS \| $c GFLOPS \|"
	done
	} \| tee calibrate_summary.md
	cat calibrate_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: calibrate-results
	path: \|
	calib.txt
	gemm_default.txt
	gemm_calibrated.txt
	calibrate_summary.md

	# ======================================================================
	# 1. PARITY — correctness before speed. if Axiom's ops diverge from TF,
	# nothing else matters. this runs first and gates the whole suite.
	# ======================================================================
	parity:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: show hardware
	run: lscpu \| grep -E '(Model name\|CPU$s$\|Thread\|Core\|Cache)' \| head -10

	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_parity

	- name: install tensorflow + numpy
	run: pip install --quiet tensorflow-cpu numpy

	- name: axiom writes .axt inputs/outputs
	run: ./build/bench_parity \| tee parity_axiom.txt

	- name: tf reads + verifies
	run: python3 benchmarks/tf_parity.py \| tee parity_tf.txt

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: parity-results
	path: \|
	parity_*.txt
	build/parity_*.axt

	# ======================================================================
	# 2. GEMM MICRO-KERNEL — square, skinny, transposed variants.
	# ======================================================================
	gemm:
	runs-on: ubuntu-latest
	needs: parity
	steps:
	- uses: actions/checkout@v4
	- name: show hardware
	run: lscpu \| grep -E '(Model\|CPU\|Cache\|MHz)' \| head -10

	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_gemm_suite

	- name: install tensorflow
	run: pip install --quiet tensorflow-cpu numpy

	- name: axiom
	run: ./build/bench_gemm_suite \| tee axiom_gemm.txt

	- name: tf
	run: python3 benchmarks/tf_gemm_suite.py \| tee tf_gemm.txt

	- name: summarize
	run: \|
	{
	echo "# GEMM suite vs TensorFlow"
	echo ""
	python3 benchmarks/summarize.py axiom_gemm.txt tf_gemm.txt --threshold $REGRESSION_PCT
	} \| tee gemm_summary.md
	cat gemm_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: gemm-results
	path: \|
	axiom_gemm.txt
	tf_gemm.txt
	gemm_summary.md

	# ======================================================================
	# 3. OPS — elementwise, reductions, softmax, BN, LN.
	# ======================================================================
	ops:
	runs-on: ubuntu-latest
	needs: parity
	steps:
	- uses: actions/checkout@v4
	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_ops_suite

	- name: install tensorflow
	run: pip install --quiet tensorflow-cpu numpy

	- name: axiom
	run: ./build/bench_ops_suite \| tee axiom_ops.txt

	- name: tf
	run: python3 benchmarks/tf_ops_suite.py \| tee tf_ops.txt

	- name: summarize
	run: \|
	{
	echo "# Ops suite vs TensorFlow"
	echo ""
	python3 benchmarks/summarize.py axiom_ops.txt tf_ops.txt --threshold $REGRESSION_PCT
	} \| tee ops_summary.md
	cat ops_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: ops-results
	path: \|
	axiom_ops.txt
	tf_ops.txt
	ops_summary.md

	# ======================================================================
	# 4. CONV — conv2d across VGG-style shapes, pooling, fused ConvBNReLU.
	# ======================================================================
	conv:
	runs-on: ubuntu-latest
	needs: parity
	steps:
	- uses: actions/checkout@v4
	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_conv_suite

	- name: install tensorflow
	run: pip install --quiet tensorflow-cpu numpy

	- name: axiom
	run: timeout 1200 ./build/bench_conv_suite \| tee axiom_conv.txt

	- name: tf
	run: timeout 1200 python3 benchmarks/tf_conv_suite.py \| tee tf_conv.txt

	- name: summarize
	run: \|
	{
	echo "# Conv suite vs TensorFlow"
	echo ""
	python3 benchmarks/summarize.py axiom_conv.txt tf_conv.txt --threshold $REGRESSION_PCT
	} \| tee conv_summary.md
	cat conv_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: conv-results
	path: \|
	axiom_conv.txt
	tf_conv.txt
	conv_summary.md

	# ======================================================================
	# 5. MHA — scaled dot-product attention, full MHA layer, KV cache.
	# uses the existing bench_mha.c + benchmarks/tf_mha.py.
	# ======================================================================
	mha:
	runs-on: ubuntu-latest
	needs: parity
	steps:
	- uses: actions/checkout@v4
	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_mha

	- name: install tensorflow
	run: pip install --quiet tensorflow-cpu numpy

	- name: axiom
	run: ./build/bench_mha 3 \| tee axiom_mha.txt

	- name: tf
	run: python3 benchmarks/tf_mha.py 3 \| tee tf_mha.txt

	- name: summarize
	run: \|
	{
	echo "# MHA suite vs TensorFlow"
	echo ""
	python3 benchmarks/summarize.py axiom_mha.txt tf_mha.txt --threshold $REGRESSION_PCT
	} \| tee mha_summary.md
	cat mha_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: mha-results
	path: \|
	axiom_mha.txt
	tf_mha.txt
	mha_summary.md

	# ======================================================================
	# 6. MODEL TRAINING — MLP (small + wide), VGG, mini transformer.
	# ======================================================================
	training:
	runs-on: ubuntu-latest
	needs: parity
	steps:
	- uses: actions/checkout@v4
	- name: show hardware
	run: lscpu \| grep -E '(Model\|CPU$s$)' && free -h

	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc)

	- name: download mnist
	run: \|
	mkdir -p examples/data && cd examples/data
	for f in train-images-idx3-ubyte train-labels-idx1-ubyte \
	t10k-images-idx3-ubyte t10k-labels-idx1-ubyte; do
	[ -f "$f" ] && continue
	curl -sL -O "https://storage.googleapis.com/cvdf-datasets/mnist/${f}.gz"
	gunzip "${f}.gz"
	done

	- name: install tensorflow
	run: pip install --quiet tensorflow-cpu

	- name: small MLP (109K)
	run: \|
	echo "=== Small MLP: Axiom ==="
	./build/ax_mnist \| tee axiom_small.txt
	echo ""
	echo "=== Small MLP: TensorFlow ==="
	python3 benchmarks/tf_mnist.py \| tee tf_small.txt

	- name: wide MLP (14.4M)
	run: \|
	echo "=== Wide MLP: Axiom ==="
	timeout 1800 ./build/bench_wide train \| tee axiom_wide.txt
	echo ""
	echo "=== Wide MLP: TensorFlow ==="
	timeout 1800 python3 benchmarks/tf_wide.py train \| tee tf_wide.txt

	- name: VGG CNN (8M)
	run: \|
	echo "=== VGG: Axiom ==="
	timeout 3600 ./build/bench_vgg train \| tee axiom_vgg.txt \|\| echo "axiom_vgg failed"
	echo ""
	echo "=== VGG: TensorFlow ==="
	timeout 3600 python3 benchmarks/tf_vgg.py \| tee tf_vgg.txt \|\| echo "tf_vgg failed"

	- name: mini Transformer
	run: \|
	echo "=== Transformer: Axiom ==="
	./build/bench_transformer \| tee axiom_tf.txt
	echo ""
	echo "=== Transformer: TensorFlow ==="
	python3 benchmarks/tf_transformer.py \| tee tf_tf.txt

	- name: summarize
	if: always()
	run: \|
	{
	echo "# Model training benchmarks"
	echo ""
	echo "## Small MLP"
	grep -h "total training time\\|per-epoch\\|test accuracy" axiom_small.txt tf_small.txt 2>/dev/null \|\| true
	echo ""
	echo "## Wide MLP (14.4M params)"
	grep -h "total training time\\|per-epoch\\|test accuracy" axiom_wide.txt tf_wide.txt 2>/dev/null \|\| true
	echo ""
	echo "## VGG CNN (8M params)"
	grep -h "total training time\\|per-epoch\\|test accuracy" axiom_vgg.txt tf_vgg.txt 2>/dev/null \|\| true
	echo ""
	echo "## Mini Transformer"
	python3 benchmarks/summarize.py axiom_tf.txt tf_tf.txt --threshold $REGRESSION_PCT \|\| true
	} \| tee training_summary.md
	cat training_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: training-results
	path: \|
	axiom_*.txt
	tf_*.txt
	training_summary.md

	# ======================================================================
	# 7. STRESS — huge GEMMs, long-context attention, thread scaling, Adam.
	# ======================================================================
	stress:
	runs-on: ubuntu-latest
	needs: parity
	steps:
	- uses: actions/checkout@v4
	- name: show hardware
	run: lscpu \| grep -E '(Model\|CPU\|MHz)' \| head -5 && free -h

	- name: build
	run: \|
	mkdir build && cd build
	cmake -DCMAKE_BUILD_TYPE=Release -DAX_CPU_ISA_DISPATCH=ON ..
	cmake --build . -j$(nproc) --target bench_stress

	- name: install tensorflow
	run: pip install --quiet tensorflow-cpu numpy

	- name: axiom
	run: timeout 1200 ./build/bench_stress \| tee axiom_stress.txt

	- name: tf
	run: timeout 1200 python3 benchmarks/tf_stress.py \| tee tf_stress.txt

	- name: summarize
	run: \|
	{
	echo "# Stress suite vs TensorFlow"
	echo ""
	echo "_huge GEMMs, long-context SDPA, Adam at scale_"
	echo ""
	python3 benchmarks/summarize.py axiom_stress.txt tf_stress.txt --threshold $REGRESSION_PCT
	} \| tee stress_summary.md
	cat stress_summary.md >> $GITHUB_STEP_SUMMARY

	- name: upload
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: stress-results
	path: \|
	axiom_stress.txt
	tf_stress.txt
	stress_summary.md

	# ======================================================================
	# 8. OVERALL — roll up all summaries into a single artifact for easy
	# PR-comment rendering. depends on all others.
	# ======================================================================
	overall:
	runs-on: ubuntu-latest
	needs: [calibrate, parity, gemm, ops, conv, mha, training, stress]
	if: always()
	steps:
	- uses: actions/checkout@v4
	- uses: actions/download-artifact@v4
	with:
	path: artifacts

	- name: build combined report
	run: \|
	{
	echo "# Axiom benchmark report"
	echo ""
	echo "all suites below compare Axiom against TensorFlow (CPU, oneDNN)"
	echo "on the same machine, same shapes, same iteration counts."
	echo ""
	for md in artifacts//calibrate_summary.md artifacts//gemm_summary.md \
	artifacts/*/ops_summary.md \
	artifacts//conv_summary.md artifacts//mha_summary.md \
	artifacts//training_summary.md artifacts//stress_summary.md; do
	[ -f "$md" ] \|\| continue
	cat "$md"
	echo ""
	done
	} > REPORT.md
	cat REPORT.md >> $GITHUB_STEP_SUMMARY

	- uses: actions/upload-artifact@v4
	with:
	name: full-report
	path: REPORT.md

Navigation Menu

docs(types): fix axiom/*.h typo causing -Wcomment #43

Workflow file

docs(types): fix axiom/*.h typo causing -Wcomment #43

Uh oh!

Workflow file for this run