|
| 1 | +"""Comprehensive real-world solver benchmark for package-maximizer.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import json |
| 6 | +import os |
| 7 | +import re |
| 8 | +import subprocess |
| 9 | +import sys |
| 10 | +import time |
| 11 | +from pathlib import Path |
| 12 | +from typing import Any |
| 13 | + |
| 14 | +from package_maximizer.core.maximizer import PackageMaximizer |
| 15 | +from package_maximizer.core.model_encoder import encode_packages |
| 16 | +from package_maximizer.core.package import Package |
| 17 | +from package_maximizer.solvers import SOLVER_REGISTRY |
| 18 | + |
| 19 | +NIXPKGS_ALL_PACKAGES = ( |
| 20 | + "/nix/store/cc7ff4ysismx0c3778v8gc6b14plrz3z-source/pkgs/top-level/all-packages.nix" |
| 21 | +) |
| 22 | + |
| 23 | + |
| 24 | +def parse_nixpkgs_packages() -> list[dict[str, str]]: |
| 25 | + """Parse nixpkgs all-packages.nix for package definitions.""" |
| 26 | + packages: list[dict[str, str]] = [] |
| 27 | + if not os.path.exists(NIXPKGS_ALL_PACKAGES): |
| 28 | + return packages |
| 29 | + with open(NIXPKGS_ALL_PACKAGES, errors="ignore") as f: |
| 30 | + content = f.read() |
| 31 | + for pattern in [ |
| 32 | + r"(\w+)\s*=\s*lib\.callPackage\s*\(", |
| 33 | + r"(\w+)\s*=\s*callPackage\s*\(", |
| 34 | + ]: |
| 35 | + for match in re.finditer(pattern, content): |
| 36 | + name = match.group(1) |
| 37 | + if name and not name.startswith("_") and len(name) > 2: |
| 38 | + packages.append({"name": name, "version": ""}) |
| 39 | + seen: set[str] = set() |
| 40 | + unique: list[dict[str, str]] = [] |
| 41 | + for pkg in packages: |
| 42 | + if pkg["name"] not in seen: |
| 43 | + seen.add(pkg["name"]) |
| 44 | + unique.append(pkg) |
| 45 | + return unique |
| 46 | + |
| 47 | + |
| 48 | +def get_nix_store_packages() -> list[dict[str, str]]: |
| 49 | + """Get packages from the Nix store with versions.""" |
| 50 | + packages: list[dict[str, str]] = [] |
| 51 | + for item in os.listdir("/nix/store"): |
| 52 | + if "-" not in item or item.endswith(".drv") or item.endswith(".patch"): |
| 53 | + continue |
| 54 | + parts = item.split("-") |
| 55 | + if len(parts) >= 2: |
| 56 | + for i in range(len(parts) - 1, 0, -1): |
| 57 | + if re.match(r"^\d", parts[i]): |
| 58 | + name = "-".join(parts[:i]) |
| 59 | + version = parts[i] |
| 60 | + packages.append({"name": name, "version": version}) |
| 61 | + break |
| 62 | + return packages |
| 63 | + |
| 64 | + |
| 65 | +def get_dependencies_for_package(package_name: str) -> list[str]: |
| 66 | + """Get dependencies for a package via nix-store --references.""" |
| 67 | + deps: list[str] = [] |
| 68 | + try: |
| 69 | + result = subprocess.run( |
| 70 | + ["nix-store", "-qR", f"/nix/store/{package_name}"], |
| 71 | + capture_output=True, |
| 72 | + text=True, |
| 73 | + timeout=10, |
| 74 | + ) |
| 75 | + if result.returncode == 0: |
| 76 | + for line in result.stdout.strip().split("\n"): |
| 77 | + line = line.strip() |
| 78 | + if line: |
| 79 | + dep = _parse_store_path(line) |
| 80 | + if dep and dep != package_name: |
| 81 | + deps.append(dep) |
| 82 | + except (subprocess.TimeoutExpired, FileNotFoundError): |
| 83 | + pass |
| 84 | + return list(dict.fromkeys(deps))[:10] |
| 85 | + |
| 86 | + |
| 87 | +def _parse_store_path(path: str) -> str | None: |
| 88 | + """Extract package name from store path.""" |
| 89 | + basename = path.rstrip("/").split("/")[-1] |
| 90 | + match = re.match(r"^[a-z0-9]+-(.+?)-(\d+\.\d+)", basename) |
| 91 | + if match: |
| 92 | + return match.group(1) |
| 93 | + return None |
| 94 | + |
| 95 | + |
| 96 | +def build_real_package_set(max_packages: int = 50) -> list[Package]: |
| 97 | + """Build a real package set from nixpkgs with dependencies.""" |
| 98 | + packages: list[Package] = [] |
| 99 | + all_names: set[str] = set() |
| 100 | + for pkg_info in parse_nixpkgs_packages()[:max_packages]: |
| 101 | + name = pkg_info["name"] |
| 102 | + if name not in all_names and len(name) < 100: |
| 103 | + all_names.add(name) |
| 104 | + packages.append(Package(name=name)) |
| 105 | + for pkg_info in get_nix_store_packages()[:max_packages]: |
| 106 | + name = pkg_info["name"] |
| 107 | + if name not in all_names and len(name) < 100: |
| 108 | + all_names.add(name) |
| 109 | + packages.append(Package(name=name, version=pkg_info["version"])) |
| 110 | + for pkg in packages[: min(20, len(packages))]: |
| 111 | + deps = get_dependencies_for_package(pkg.name) |
| 112 | + if deps: |
| 113 | + pkg.depends = deps |
| 114 | + return packages |
| 115 | + |
| 116 | + |
| 117 | +def run_benchmark( |
| 118 | + packages: list[Package], |
| 119 | + solver_names: list[str] | None = None, |
| 120 | + max_packages: int = 50, |
| 121 | +) -> dict[str, Any]: |
| 122 | + """Run all solvers and collect metrics.""" |
| 123 | + if solver_names is None: |
| 124 | + solver_names = list(SOLVER_REGISTRY.keys()) |
| 125 | + results: dict[str, Any] = {} |
| 126 | + comparison: dict[str, dict] = {} |
| 127 | + total_input = len(packages) |
| 128 | + for solver_name in solver_names: |
| 129 | + if solver_name not in SOLVER_REGISTRY: |
| 130 | + comparison[solver_name] = {"status": "SKIPPED", "reason": f"Unknown solver"} |
| 131 | + continue |
| 132 | + maximizer = PackageMaximizer(manager="apt", solver=solver_name) |
| 133 | + start_time = time.perf_counter() |
| 134 | + try: |
| 135 | + selected = maximizer.maximize(packages[:max_packages]) |
| 136 | + elapsed = time.perf_counter() - start_time |
| 137 | + selected_names = set(p.name for p in selected) |
| 138 | + selected_count = len(selected) |
| 139 | + accuracy = selected_count / max_packages * 100 if max_packages > 0 else 0 |
| 140 | + constraints = encode_packages(packages[:max_packages]) |
| 141 | + conflicts_in_selected = sum( |
| 142 | + 1 |
| 143 | + for a, b in constraints.conflicts |
| 144 | + if a in selected_names and b in selected_names |
| 145 | + ) |
| 146 | + deps_unmet = sum( |
| 147 | + 1 |
| 148 | + for pkg in selected |
| 149 | + for dep in (pkg.depends or []) |
| 150 | + if dep not in selected_names |
| 151 | + ) |
| 152 | + comparison[solver_name] = { |
| 153 | + "status": "OK", |
| 154 | + "time_seconds": round(elapsed, 4), |
| 155 | + "packages_selected": selected_count, |
| 156 | + "selection_rate": round(accuracy, 1), |
| 157 | + "conflicts_in_selected": conflicts_in_selected, |
| 158 | + "unmet_dependencies": deps_unmet, |
| 159 | + "correctness_score": round( |
| 160 | + (1 - conflicts_in_selected / max(1, selected_count)) |
| 161 | + * (1 - deps_unmet / max(1, selected_count)) |
| 162 | + * 100, |
| 163 | + 1, |
| 164 | + ), |
| 165 | + "accuracy": round(accuracy, 1), |
| 166 | + } |
| 167 | + except Exception as e: |
| 168 | + elapsed = time.perf_counter() - start_time |
| 169 | + comparison[solver_name] = { |
| 170 | + "status": "ERROR", |
| 171 | + "time_seconds": round(elapsed, 4), |
| 172 | + "error": str(e)[:100], |
| 173 | + } |
| 174 | + successful = {k: v for k, v in comparison.items() if v.get("status") == "OK"} |
| 175 | + if successful: |
| 176 | + fastest = min(successful.items(), key=lambda x: x[1]["time_seconds"]) |
| 177 | + most_accurate = max(successful.items(), key=lambda x: x[1]["accuracy"]) |
| 178 | + most_correct = max(successful.items(), key=lambda x: x[1]["correctness_score"]) |
| 179 | + highest_selection = max( |
| 180 | + successful.items(), key=lambda x: x[1]["packages_selected"] |
| 181 | + ) |
| 182 | + results["summary"] = { |
| 183 | + "fastest_solver": fastest[0], |
| 184 | + "fastest_time": fastest[1]["time_seconds"], |
| 185 | + "most_accurate_solver": most_accurate[0], |
| 186 | + "most_accurate_rate": most_accurate[1]["accuracy"], |
| 187 | + "most_correct_solver": most_correct[0], |
| 188 | + "most_correct_score": most_correct[1]["correctness_score"], |
| 189 | + "highest_selection_solver": highest_selection[0], |
| 190 | + "highest_selection_count": highest_selection[1]["packages_selected"], |
| 191 | + "total_input_packages": total_input, |
| 192 | + "solvers_tested": len(successful), |
| 193 | + } |
| 194 | + results["solvers"] = comparison |
| 195 | + return results |
| 196 | + |
| 197 | + |
| 198 | +def print_report(results: dict[str, Any]) -> None: |
| 199 | + """Print a formatted benchmark report.""" |
| 200 | + print(f"\n{'='*70}") |
| 201 | + print(" PACKAGE MAXIMIZER — SOLVER BENCHMARK REPORT") |
| 202 | + print(f"{'='*70}") |
| 203 | + if "summary" in results: |
| 204 | + s = results["summary"] |
| 205 | + print(f"\n Total Input Packages: {s['total_input_packages']}") |
| 206 | + print(f" Solvers Tested: {s['solvers_tested']}") |
| 207 | + print(f"\n Fastest: {s['fastest_solver']} ({s['fastest_time']}s)") |
| 208 | + print( |
| 209 | + f" Most Accurate: {s['most_accurate_solver']} ({s['most_accurate_rate']}%)" |
| 210 | + ) |
| 211 | + print( |
| 212 | + f" Most Correct: {s['most_correct_solver']} ({s['most_correct_score']}%)" |
| 213 | + ) |
| 214 | + print( |
| 215 | + f" Highest Selection: {s['highest_selection_solver']} ({s['highest_selection_count']} pkgs)" |
| 216 | + ) |
| 217 | + print(f"\n{'='*70}") |
| 218 | + print(f" DETAILED RESULTS") |
| 219 | + print(f"{'='*70}") |
| 220 | + print( |
| 221 | + f"\n {'Solver':<20s} {'Status':<8s} {'Time':<10s} {'Selected':<10s} {'Accuracy':<10s} {'Correct':<10s}" |
| 222 | + ) |
| 223 | + print(f" {'-'*20} {'-'*8} {'-'*10} {'-'*10} {'-'*10} {'-'*10}") |
| 224 | + for name, data in results["solvers"].items(): |
| 225 | + status = data.get("status", "?") |
| 226 | + if status == "OK": |
| 227 | + print( |
| 228 | + f" {name:<20s} {status:<8s} " |
| 229 | + f"{data['time_seconds']:<10.4f} " |
| 230 | + f"{data['packages_selected']:<10d} " |
| 231 | + f"{data['accuracy']:<10.1f} " |
| 232 | + f"{data['correctness_score']:<10.1f}" |
| 233 | + ) |
| 234 | + else: |
| 235 | + print( |
| 236 | + f" {name:<20s} {status:<8s} {'-':<10s} {'-':<10s} {'-':<10s} {'-':<10s}" |
| 237 | + ) |
| 238 | + print(f"\n{'='*70}") |
| 239 | + print(f" DEPENDENCY ANALYSIS") |
| 240 | + print(f"{'='*70}") |
| 241 | + for name, data in results["solvers"].items(): |
| 242 | + if data.get("status") == "OK": |
| 243 | + print( |
| 244 | + f" {name:<20s}: {data['conflicts_in_selected']} conflicts, " |
| 245 | + f"{data['unmet_dependencies']} unmet deps" |
| 246 | + ) |
| 247 | + print() |
| 248 | + |
| 249 | + |
| 250 | +def run_full_benchmark( |
| 251 | + max_packages: int = 50, |
| 252 | + solver_names: list[str] | None = None, |
| 253 | + verbose: bool = False, |
| 254 | +) -> dict[str, Any]: |
| 255 | + """Full benchmark: build real package set, run all solvers, report.""" |
| 256 | + print(f"\n{'='*70}") |
| 257 | + print(" REAL-WORLD DATA BENCHMARK") |
| 258 | + print(f"{'='*70}") |
| 259 | + print(f"\n[1/2] Building package set from nixpkgs...") |
| 260 | + packages = build_real_package_set(max_packages) |
| 261 | + print(f" Built {len(packages)} packages with real dependencies") |
| 262 | + print(f"\n[2/2] Running solvers...") |
| 263 | + results = run_benchmark(packages, solver_names, max_packages) |
| 264 | + print_report(results) |
| 265 | + output_path = Path("/home/domini/package-maximizer/benchmark_results.json") |
| 266 | + with open(output_path, "w") as f: |
| 267 | + json.dump(results, f, indent=2, default=str) |
| 268 | + print(f" Results saved to {output_path}") |
| 269 | + return results |
| 270 | + |
| 271 | + |
| 272 | +if __name__ == "__main__": |
| 273 | + import argparse |
| 274 | + |
| 275 | + parser = argparse.ArgumentParser( |
| 276 | + description="Benchmark package-maximizer solvers on real data" |
| 277 | + ) |
| 278 | + parser.add_argument("--max-packages", type=int, default=50) |
| 279 | + parser.add_argument("--solvers", nargs="+", default=None) |
| 280 | + parser.add_argument("--verbose", "-v", action="store_true") |
| 281 | + args = parser.parse_args() |
| 282 | + results = run_full_benchmark( |
| 283 | + max_packages=args.max_packages, |
| 284 | + solver_names=args.solvers, |
| 285 | + verbose=args.verbose, |
| 286 | + ) |
| 287 | + solvers_ok = all(v.get("status") == "OK" for v in results["solvers"].values()) |
| 288 | + sys.exit(0 if solvers_ok else 1) |
0 commit comments