Skip to content

Commit aa0fd83

Browse files
Thesola10Thesola10
committed
Introducing NixOS end-to-end test framework
The time has finally come! This system should allow interactive tests by simulating the mouse and keyboard and comparing against screenshots (and maybe also ensure GNOME hasn't crashed).
1 parent 8e038d0 commit aa0fd83

File tree

7 files changed

+157
-29
lines changed

7 files changed

+157
-29
lines changed

‎flake.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎flake.nix‎

Lines changed: 47 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,35 @@
44
inputs."nixpkgs-gnome".url = github:NixOS/nixpkgs/wip-gnome;
55

66
outputs = { self, nixpkgs, nixpkgs-gnome, flake-utils, ... }:
7+
let
8+
testSystem = "x86_64-linux";
9+
pkgs-gnome = import nixpkgs-gnome { system = testSystem; };
10+
gnomeOverlay = (s: super: {
11+
gnome-desktop = pkgs-gnome.gnome-desktop;
12+
gnome-shell = pkgs-gnome.gnome-shell.override {
13+
evolution-data-server-gtk4 = super.evolution-data-server-gtk4.override {
14+
inherit (super) webkitgtk_4_1 webkitgtk_6_0;
15+
};
16+
};
17+
gnome-session = pkgs-gnome.gnome-session.override {
18+
inherit (s) gnome-shell;
19+
};
20+
gnome-control-center = pkgs-gnome.gnome-control-center;
21+
gnome-initial-setup = pkgs-gnome.gnome-initial-setup.override {
22+
inherit (super) webkitgtk_6_0;
23+
};
24+
gnome-settings-daemon = pkgs-gnome.gnome-settings-daemon;
25+
mutter = pkgs-gnome.mutter;
26+
gdm = pkgs-gnome.gdm;
27+
xdg-desktop-portal-gnome = pkgs-gnome.xdg-desktop-portal-gnome;
28+
xdg-desktop-portal-gtk = pkgs-gnome.xdg-desktop-portal-gtk;
29+
});
30+
31+
# NixOS usually takes its sweet time updating to latest GNOME.
32+
# Enable this to use the GNOME version from their dedicated dev branch.
33+
#WARN: build times may increase significantly!
34+
useGnomeStaging = false;
35+
in
736
flake-utils.lib.eachDefaultSystem
837
(system:
938
let hostPkgs = import nixpkgs { inherit system; };
@@ -21,40 +50,32 @@
2150
})
2251
];
2352
};
24-
in localConfig.config.system.build.vm;
53+
in localConfig.config.system.build.vm;
54+
55+
checks = import ./tests {
56+
system = testSystem;
57+
pkgs = import nixpkgs { system = testSystem; };
58+
defaultConfig = {
59+
imports = [ ./vm.nix ];
60+
nixpkgs.overlays = [
61+
(s: super: { paperwm = self.packages.${testSystem}.default; })
62+
63+
(if useGnomeStaging then gnomeOverlay else (s: super: {}))
64+
];
65+
};
66+
};
2567
}) // {
2668
nixosConfigurations."testbox" =
27-
let system = "x86_64-linux";
28-
pkgs-gnome = import nixpkgs-gnome { inherit system; };
29-
in nixpkgs.lib.nixosSystem {
30-
inherit system;
69+
nixpkgs.lib.nixosSystem {
70+
system = testSystem;
3171
modules = [
3272
./vm.nix
3373
{ nixpkgs.overlays = [
3474
# Introduce PaperWM into our extensions
35-
(s: super: { paperwm = self.packages.${system}.default; })
75+
(s: super: { paperwm = self.packages.${testSystem}.default; })
3676

3777
# Pull GNOME-specific packages from GNOME staging
38-
#(s: super: {
39-
# gnome-desktop = pkgs-gnome.gnome-desktop;
40-
# gnome-shell = pkgs-gnome.gnome-shell.override {
41-
# evolution-data-server-gtk4 = super.evolution-data-server-gtk4.override {
42-
# inherit (super) webkitgtk_4_1 webkitgtk_6_0;
43-
# };
44-
# };
45-
# gnome-session = pkgs-gnome.gnome-session.override {
46-
# inherit (s) gnome-shell;
47-
# };
48-
# gnome-control-center = pkgs-gnome.gnome-control-center;
49-
# gnome-initial-setup = pkgs-gnome.gnome-initial-setup.override {
50-
# inherit (super) webkitgtk_6_0;
51-
# };
52-
# gnome-settings-daemon = pkgs-gnome.gnome-settings-daemon;
53-
# mutter = pkgs-gnome.mutter;
54-
# gdm = pkgs-gnome.gdm;
55-
# xdg-desktop-portal-gnome = pkgs-gnome.xdg-desktop-portal-gnome;
56-
# xdg-desktop-portal-gtk = pkgs-gnome.xdg-desktop-portal-gtk;
57-
#})
78+
(if useGnomeStaging then gnomeOverlay else (s: super: {}))
5879
];
5980
}
6081
];

‎tests/README.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# PaperWM end-to-end tests
2+
3+
Unit tests on PaperWM are carried out using the [test VM](https://github.com/PaperWM/PaperWM/wiki/Using-the-test-VM), a NixOS virtual machine connected to a Python testing harness, and [Behave](https://behave.readthedocs.io), an implementation of the Gherkin unit testing language.
4+
5+
## Layout
6+
7+
### `default.nix` and `template.nix`
8+
9+
These files are NixOS boilerplate code that scan for features in the `features` directory and generate test VMs accordingly. You can use them to register tests that require extra dependencies not found in the base test VM, such as specific misbehaving applications.
10+
11+
### `features`
12+
13+
This is the directory where Behave feature tests and their Python implementations are found. Each `.feature` file maps to a NixOS unit test, which can be overridden in `default.nix` as mentioned above.
14+
15+
The usual testing loop is to record user input from within the VM, and replay them in the order specified in the feature file.
16+
17+
### `recordings`
18+
19+
This directory contains recordings of all input devices in the VM (touchpad, touchscreen, keyboard...), meant to be replayed as-is in the VM by the feature tests. Please only add recordings made from within the test VM.
20+
21+
### `screenshots`
22+
23+
This directory contains predicate screenshots to check against, to determine whether a test was successful.

‎tests/default.nix‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{ pkgs ? import <nixpkgs> {}
2+
, system ? builtins.currentSystem
3+
, runTest ? pkgs.testers.nixosTest
4+
, defaultConfig ?
5+
{ ... }: { imports = [ ../vm.nix ]; }
6+
, ... }:
7+
8+
#
9+
# Root file for the set of unit tests on PaperWM. These make use of the NixOS
10+
# unit test framework, and individual tests can be registered as Behave feature
11+
# files.
12+
# Documentation on the unit test framework is available in the NixOS manual:
13+
#
14+
# https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
15+
#
16+
17+
let
18+
lib = pkgs.lib;
19+
20+
# Run a single Behave feature file
21+
behaveTest = featureName: as: runTest
22+
(import ./template.nix ({ inherit defaultConfig pkgs featureName; } // as));
23+
24+
allBehaveTests =
25+
let allBehaveFiles =
26+
lib.filterAttrs (k: v: lib.hasSuffix ".feature" k && v == "regular")
27+
(builtins.readDir ./features);
28+
testName = s: lib.removeSuffix ".feature" s;
29+
in builtins.foldl'
30+
(l: r: l // { "${testName r}" = behaveTest r {}; })
31+
{} (builtins.attrNames allBehaveFiles);
32+
33+
in allBehaveTests //
34+
{
35+
# Tests that require a specific system configuration go here
36+
}
37+
38+
# Note: To run an individual unit test automatically (as part of the suite),
39+
# run the following:
40+
# nix build .#checks.x86_64-linux.(test name)
41+
#
42+
# To debug a test (i.e. run it interactively), the command isn't very different
43+
# nix run .#checks.x86_64-linux.(test name).driverInteractive
44+

‎tests/features/basic.feature‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Feature: The test machine starts
2+
3+
Scenario: The test machine starts
4+
When the machine starts
5+
Then the machine should reach graphics

‎tests/features/steps/basic.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from behave import given, when, then
2+
from types import SimpleNamespace
3+
4+
@when("the machine starts")
5+
def machine_boot(context):
6+
nixos = SimpleNamespace(**context.config.userdata)
7+
# no-op: our test template starts the machine already
8+
9+
@then("the machine should reach graphics")
10+
def graphical_target(context):
11+
nixos = SimpleNamespace(**context.config.userdata)
12+
nixos.machine.wait_for_unit("graphical.target")

‎tests/template.nix‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{ pkgs, defaultConfig
2+
, testsDir ? ./.
3+
, featureName
4+
, ... }@opts:
5+
6+
{
7+
name = featureName;
8+
nodes = { machine = defaultConfig; };
9+
10+
extraPythonPackages = p: with p; [ behave ];
11+
12+
skipTypeCheck = true;
13+
14+
testScript = ''
15+
from behave.configuration import Configuration
16+
from behave.__main__ import run_behave
17+
18+
conf = Configuration("${testsDir}/features/${featureName}", userdata = driver.test_symbols())
19+
start_all()
20+
if run_behave(conf) != 0:
21+
raise AssertionError("One or more Behave features have failed. Check the logs above for details.")
22+
'';
23+
} // (pkgs.lib.filterAttrs (n: v: n != "pkgs" && n != "defaultConfig" && n != "testsDir" && n != "featureName") opts)

0 commit comments

Comments
 (0)