feat: Add Docker support to integration tests

This allows the tests to be run inside of a Docker container with linux
with X11 inside.

Signed-off-by: John Nunley <dev@notgull.net>

.github/CODEOWNERS

@@ -32,3 +32,6 @@
 # Orbital (Redox OS)
 /src/platform/orbital.rs            @jackpot51
 /src/platform_impl/orbital          @jackpot51
+
+# Integration tests
+/it                                 @notgull

.github/workflows/ci.yml

@@ -202,6 +202,41 @@ jobs:
           ~/.cargo/git/db/
         key: cargo-${{ matrix.toolchain }}-${{ matrix.platform.name }}-${{ hashFiles('Cargo.lock') }}
 
+  it:
+    name: Run integration tests on ${{ matrix.platform.name }}
+    runs-on: ${{ matrix.platform.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        toolchain: [stable, nightly]
+        platform:
+          # Note: Make sure that we test all the `docs.rs` targets defined in Cargo.toml!
+          - { name: 'X11',                target: x86_64-unknown-linux-gnu, os: ubuntu-latest, options: '--no-default-features --features=x11' }
+
+    env:
+      # Set more verbose terminal output
+      CARGO_TERM_VERBOSE: true
+      RUST_BACKTRACE: 1
+
+      # Faster compilation and error on warnings
+      RUSTFLAGS: '--codegen=debuginfo=0 --deny=warnings ${{ matrix.platform.rustflags }}'
+
+      OPTIONS: --target=${{ matrix.platform.target }} ${{ matrix.platform.options }}
+
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dtolnay/rust-toolchain@master
+      with:
+        toolchain: ${{ matrix.toolchain }}
+
+    - name: Log into GHCR
+      run: |
+        echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+    - name: Common tests
+      run: cargo run -p gui-test-runner -- common-tests ${{ matrix.platform.target }}
+
   cargo-deny:
     name: Run cargo-deny on ${{ matrix.platform.name }}
     runs-on: ubuntu-latest

Cargo.toml

@@ -281,4 +281,5 @@ async-io = "2.3.1"
 gui-test = { path = "it/gui-test" }
 mint = "0.5.6"
 serde = { version = "1", features = ["serde_derive"] }
+serde_json = "1.0.114"
 winit = { path = "." }

dockerfiles/Dockerfile.ubuntu

@@ -0,0 +1,40 @@
+# syntax=docker/dockerfile:1
+# Copyright 2024 The Winit Contributors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG DISTRO=ubuntu
+ARG DISTRO_VERSION=22.04
+
+FROM "${DISTRO}":"${DISTRO_VERSION}"
+SHELL ["/bin/bash", "-eEuxo", "pipefail", "-c"]
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN \
+apt-get -o Acquire::Retries=10 -qq update && \
+apt-get -o Acquire::Retries=10 -o Dpkg::Use-Pty=0 install -y --no-install-recommends \
+    cargo \
+    ca-certificates \
+    libx11-dev \
+    libxcursor-dev \
+    libxcb1-dev \
+    libxi-dev \
+    libxkbcommon-dev \
+    libxkbcommon-x11-dev \
+    xvfb && \
+rm -rf \
+    /var/lib/apt/lists/* \
+    /var/cache/* \
+    /var/log/* \
+    /usr/share/{doc,man}
+

it/common-tests/src/main.rs

@@ -5,15 +5,17 @@ use macro_rules_attribute::apply;
 
 use winit::event_loop::EventLoop;
 
+#[allow(deprecated)]
 #[apply(test)]
 fn initialize(harness: &mut Harness) {
-    let _test = harness.test("startup/shutdown");
-
-    let evl = EventLoop::new().unwrap();
-    evl.run(|_event, elwt| {
-        elwt.exit();
-    })
-    .unwrap();
+    let mut group = harness.group("sanity");
+    group.harness().with_test("startup/shutdown", || {
+        let evl = EventLoop::new().expect("initialization");
+        evl.run(|_event, elwt| {
+            elwt.exit();
+        })
+        .expect("running");
+    });
 }
 
 gui_test::main! {

it/gui-test-runner/Cargo.toml

@@ -7,3 +7,7 @@ license.workspace = true
 edition.workspace = true
 
 [dependencies]
+camino = "1.1.6"
+fastrand = "2.0.1"
+gui-test.workspace = true
+serde_json.workspace = true

it/gui-test-runner/src/command.rs

@@ -0,0 +1,81 @@
+// Copyright 2024 The Winit Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! A wrapper around the `Command` type that dumps the command to stderr.
+//!
+//! Essentially it's like `set -x` in Bash.
+
+use std::ffi::{OsStr, OsString};
+use std::io::{self, prelude::*};
+use std::process::Child;
+
+/// Simple `Command` wrapper.
+pub(super) struct Command {
+    /// Actual inner command.
+    inner: std::process::Command,
+
+    /// Command to run.
+    text: Vec<OsString>,
+}
+
+impl Command {
+    /// Create a new `Command`.
+    pub(super) fn new(cmd: impl AsRef<OsStr>) -> Self {
+        let cmd = cmd.as_ref();
+        Self {
+            inner: std::process::Command::new(cmd),
+            text: vec![cmd.to_os_string()],
+        }
+    }
+
+    /// Add an argument to the `Command`.
+    pub(super) fn arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
+        let arg = arg.as_ref();
+        self.inner.arg(arg);
+        self.text.push(arg.to_os_string());
+        self
+    }
+
+    /// Add multiple arguments to the `Command`.
+    pub(super) fn args<T: AsRef<OsStr>>(&mut self, args: impl IntoIterator<Item = T>) -> &mut Self {
+        for arg in args {
+            let arg = arg.as_ref();
+            self.inner.arg(arg);
+            self.text.push(arg.to_os_string());
+        }
+
+        self
+    }
+
+    /// Spawn the process.
+    pub(super) fn spawn(&mut self) -> io::Result<Child> {
+        dump_text(&self.text);
+        self.inner.spawn()
+    }
+}
+
+/// Dump `OsString` list to stderr.
+fn dump_text(text: &[OsString]) {
+    let mut cerr = io::stderr().lock();
+    write!(&mut cerr, "+").unwrap();
+
+    for arg in text {
+        match arg.to_str() {
+            Some(arg) => write!(&mut cerr, " {}", arg).unwrap(),
+            None => write!(&mut cerr, " {:?}", arg).unwrap(),
+        }
+    }
+
+    writeln!(&mut cerr).unwrap();
+}

it/gui-test-runner/src/docker/command.rs

@@ -0,0 +1,91 @@
+// Copyright 2024 The Winit Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Run the actual Docker command.
+
+use crate::command::Command;
+
+use camino::Utf8Path;
+
+use std::ffi::OsStr;
+use std::io;
+use std::process::Child;
+
+/// The Docker command line.
+pub(super) struct DockerRun {
+    command: Command,
+}
+
+impl DockerRun {
+    /// Start the command.
+    pub(super) fn new() -> Self {
+        let mut command = Command::new("docker");
+        command.arg("run");
+
+        Self { command }
+    }
+
+    /// Run with an environment variable.
+    pub(super) fn env(&mut self, name: impl AsRef<str>, value: impl AsRef<str>) -> &mut Self {
+        let env_arg = format!("{}={}", name.as_ref(), value.as_ref());
+        self.command.args(["--env", &env_arg]);
+        self
+    }
+
+    /// Run with a simple `init` process.
+    pub(super) fn init(&mut self) -> &mut Self {
+        self.command.arg("--init");
+        self
+    }
+
+    /// Set the working directory.
+    pub(super) fn workdir(&mut self, dir: impl AsRef<OsStr>) -> &mut Self {
+        self.command.arg("--workdir");
+        self.command.arg(dir);
+        self
+    }
+
+    /// Remove the container once it is complete.
+    pub(super) fn rm(&mut self) -> &mut Self {
+        self.command.arg("--rm");
+        self
+    }
+
+    /// Pass a volume into the container.
+    pub(super) fn volume(
+        &mut self,
+        host: impl AsRef<Utf8Path>,
+        container: impl AsRef<Utf8Path>,
+    ) -> &mut Self {
+        let list = format!("{}:{}", host.as_ref(), container.as_ref());
+        self.command.args(["--volume", &list]);
+        self
+    }
+
+    /// Run the container with a command.
+    pub(super) fn run_with_command<T: AsRef<OsStr>>(
+        &mut self,
+        container_name: impl AsRef<str>,
+        container_version: impl AsRef<str>,
+        command: impl IntoIterator<Item = T>,
+    ) -> io::Result<Child> {
+        self.command.arg(format!(
+            "{}:{}",
+            container_name.as_ref(),
+            container_version.as_ref()
+        ));
+        self.command.args(command);
+        self.command.spawn()
+    }
+}

it/gui-test-runner/src/docker/linux.rs

@@ -0,0 +1,114 @@
+// Copyright 2024 The Winit Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Run tests inside of a Linux docker container.
+
+use super::command::DockerRun;
+use crate::stream::StreamReader;
+
+use gui_test::remote::handler;
+use gui_test::TestHandler;
+
+use std::io;
+use std::os::unix::net::UnixListener;
+use std::path::Path;
+use std::thread;
+
+const UBUNTU_DOCKERFILE: &str = "ghcr.io/rust-windowing/testubuntu";
+const LATEST: &str = "latest";
+
+/// Run the provided test in a Linux docker container.
+pub(crate) fn linux_test(test_name: &str) -> io::Result<()> {
+    // Create a Unix socket to listen for events on.
+    let unix_path = format!("/tmp/gui_test_{}.sock", fastrand::u16(..));
+    let listener = UnixListener::bind(&unix_path)?;
+
+    // Spawn the Docker container.
+    let mut container = {
+        let mut docker = DockerRun::new();
+
+        // Usual options.
+        docker.rm().init();
+
+        // Pass through the socket as a volume.
+        docker.volume(&unix_path, &unix_path);
+
+        // Pass through the winit directory.
+        let winit_directory = Path::new(env!("CARGO_MANIFEST_DIR"))
+            .ancestors()
+            .find_map(|path| {
+                let cargo_toml = path.join("Cargo.toml");
+                let contents = std::fs::read(cargo_toml).ok()?;
+
+                if std::str::from_utf8(&contents)
+                    .ok()?
+                    .contains("name = \"winit\"")
+                {
+                    Some(path)
+                } else {
+                    None
+                }
+            })
+            .unwrap();
+        docker.volume(
+            camino::Utf8Path::from_path(winit_directory).unwrap(),
+            "/app/winit/",
+        );
+
+        // Set the working dir to this directory.
+        docker.workdir("/app/winit/");
+
+        // Set GUI_TEST_UNIX_STREAM to the socket.
+        docker.env("GUI_TEST_UNIX_STREAM", &unix_path);
+
+        // Set CARGO_TARGET_DIR to a random other directory.
+        docker.env("CARGO_TARGET_DIR", "/tmp/");
+
+        // The command to run the test.
+        let command = ["xvfb-run", "cargo", "run", "-p", test_name];
+
+        // Spawn the test container.
+        docker.run_with_command(UBUNTU_DOCKERFILE, LATEST, command)?
+    };
+
+    // Run the console listener in another thread.
+    let handle = thread::spawn(move || {
+        // Attach to the listener.
+        let (event_reader, _) = listener.accept().unwrap();
+
+        // Read events and output them as we get them.
+        let input = StreamReader::new(event_reader);
+        let mut output = handler();
+
+        for event in input {
+            let event = event?;
+            output.handle_test(event);
+        }
+
+        io::Result::Ok(())
+    });
+
+    // Wait for the container to finish.
+    if !container.wait()?.success() {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            "docker exited with a failure exit code",
+        ));
+    }
+
+    // Stop the thread.
+    handle.join().unwrap().unwrap();
+
+    Ok(())
+}

it/gui-test-runner/src/docker/mod.rs

@@ -0,0 +1,20 @@
+// Copyright 2024 The Winit Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Dealing with Docker.
+
+mod command;
+
+#[cfg(unix)]
+pub(super) mod linux;

it/gui-test-runner/src/main.rs

@@ -1,5 +1,9 @@
 //! Runner for the `gui-test` system.
 
+mod command;
+mod docker;
+mod stream;
+
 use std::env;
 use std::process::{Command, Stdio};
 
@@ -23,6 +27,13 @@ fn main() {
     // Get the current target.
     let current_target = current_target();
 
+    // If we are building for Linux, run the Linux Docker container.
+    // TODO: Architecture differences.
+    if target.contains("linux") {
+        docker::linux::linux_test(&test_crate).unwrap();
+        return;
+    }
+
     // For now, we only support building for the current target.
     assert_eq!(target, current_target);
     assert!(tag.is_none());

it/gui-test-runner/src/stream.rs

@@ -0,0 +1,81 @@
+// Copyright 2024 The Winit Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Read test events from a stream.
+
+use gui_test::{TestEvent, TestEventType};
+use std::io::{self, Read};
+
+/// Read events from a stream.
+pub(super) struct StreamReader<R> {
+    /// The inner reader.
+    reader: Option<R>,
+
+    /// Reused buffer.
+    buffer: Vec<u8>,
+}
+
+impl<R: Read> StreamReader<R> {
+    /// Create a new stream reader.
+    pub(super) fn new(reader: R) -> Self {
+        Self {
+            reader: Some(reader),
+            buffer: vec![0u8; 1024],
+        }
+    }
+}
+
+macro_rules! leap {
+    ($self:expr, $e:expr) => {{
+        match ($e) {
+            Ok(x) => x,
+            Err(err) => {
+                ($self).reader = None;
+                return Some(Err(err));
+            }
+        }
+    }};
+}
+
+impl<R: Read> Iterator for StreamReader<R> {
+    type Item = io::Result<TestEvent>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let reader = self.reader.as_mut()?;
+
+        // Read eight bytes from the reader to get payload length.
+        let mut len_buffer = [0u8; 8];
+        leap!(self, reader.read_exact(&mut len_buffer));
+
+        // Parse that, then read the length's worth of bytes.
+        let length = u64::from_be_bytes(len_buffer);
+        self.buffer.resize(length as usize, 0);
+        leap!(self, reader.read_exact(&mut self.buffer));
+
+        // Parse as a test event.
+        let event: TestEvent = leap!(
+            self,
+            serde_json::from_slice(&self.buffer)
+                .map_err(|err| io::Error::new(io::ErrorKind::Other, err))
+        );
+
+        // If this is complete, stop running.
+        if matches!(event.ty, TestEventType::Complete { .. }) {
+            self.reader = None;
+        }
+
+        // We are okay.
+        Some(Ok(event))
+    }
+}

it/gui-test/Cargo.toml

@@ -14,4 +14,4 @@ async-process = "2.1.0"
 inventory = "0.3.15"
 owo-colors = "4.0.0"
 serde = { workspace = true, features = ["derive"] }
-serde_json = "1.0.114"
+serde_json.workspace = true

it/gui-test/src/lib.rs

@@ -10,10 +10,11 @@ use std::env;
 use std::ffi::{OsStr, OsString};
 use std::mem;
 use std::num::NonZeroUsize;
+use std::panic;
 
 const GUI_TEST_CURRENT_TEST_NAME: &str = "GUI_TEST_CURRENT_TEST_NAME";
 const GUI_TEST_SUBPROCESS_LIMIT: &str = "GUI_TEST_SUBPROCESS_LIMIT";
-const DEFAULT_LIMIT: usize = 4;
+const DEFAULT_LIMIT: usize = 1;
 
 #[doc(hidden)]
 pub use inventory as __inventory;
@@ -23,7 +24,7 @@ pub use inventory as __inventory;
 macro_rules! main {
     ($handler:expr) => {
         fn main() {
-            $crate::__entry($handler)
+            $crate::__entry(|| $handler)
         }
     };
 }
@@ -136,8 +137,22 @@ impl Harness {
 
     /// Run a closure as a test.
     pub fn with_test<T>(&mut self, name: impl Into<String>, f: impl FnOnce() -> T) -> T {
-        let _test = self.test(name.into());
-        f()
+        let test = self.test(name.into());
+        match panic::catch_unwind(panic::AssertUnwindSafe(f)) {
+            Ok(x) => x,
+
+            Err(err) => {
+                if let Some(panic) = err.downcast_ref::<&'static str>() {
+                    test.fail(panic.to_string());
+                } else if let Some(panic) = err.downcast_ref::<String>() {
+                    test.fail(panic.clone());
+                } else {
+                    test.fail("unintelligible error".to_string());
+                }
+
+                panic::resume_unwind(err)
+            }
+        }
     }
 
     /// Begin a test group.
@@ -238,6 +253,14 @@ impl Testing<'_> {
         // Send the "skipped" event.
         self.harness.take().unwrap().end_test(TestResult::Skipped);
     }
+
+    /// Fail this test.
+    fn fail(mut self, panic: String) {
+        self.harness
+            .take()
+            .unwrap()
+            .end_test(TestResult::Failed(panic));
+    }
 }
 
 impl Drop for Testing<'_> {
@@ -351,7 +374,7 @@ pub enum TestResult {
 
 /// Entry point of the test.
 #[doc(hidden)]
-pub fn __entry<H: TestHandler + Send + 'static>(handler: H) {
+pub fn __entry<H: TestHandler + Send + 'static>(handler: impl FnOnce() -> H) {
     // Look for the test name environment variable.
     if let Some(test_name) = env::var(GUI_TEST_CURRENT_TEST_NAME)
         .ok()
@@ -364,10 +387,13 @@ pub fn __entry<H: TestHandler + Send + 'static>(handler: H) {
             .unwrap_or_else(|| panic!("unable to find test '{test_name}'"));
 
         // Create a harness.
-        let mut harness = Harness::new(test_to_run.name, handler);
+        let mut harness = Harness::new(test_to_run.name, handler());
 
         // Run the test.
-        (test_to_run.func)(&mut harness);
+        panic::catch_unwind(panic::AssertUnwindSafe(move || {
+            (test_to_run.func)(&mut harness)
+        }))
+        .ok();
     } else {
         // Run a subprocess for every test.
         let limit = env::var(GUI_TEST_SUBPROCESS_LIMIT)

it/gui-test/src/stream.rs

@@ -8,7 +8,8 @@ use crate::{TestEvent, TestHandler};
 use std::io::Write;
 
 /// A wrapper around a writer that sends data down a stream.
-pub struct WriteHandler<W> {
+#[derive(Debug)]
+pub struct WriteHandler<W: Write> {
     /// The inner writer.
     writer: W,
 }
@@ -30,3 +31,9 @@ impl<W: Write> TestHandler for WriteHandler<W> {
         self.writer.write_all(&payload).unwrap();
     }
 }
+
+impl<W: Write> Drop for WriteHandler<W> {
+    fn drop(&mut self) {
+        self.writer.flush().ok();
+    }
+}

it/gui-test/src/user.rs

@@ -70,6 +70,7 @@ impl UserHandler {
 
             // If this is the end, dump other events.
             while ender {
+                ender = false;
                 assert!(self.current_start.take().is_some());
 
                 // Pick one set.
@@ -78,11 +79,12 @@ impl UserHandler {
                     self.current_start = Some(test_name);
 
                     // Dump events and look for a conclusion.
-                    ender = false;
                     self.dump_events(entries.into_iter().inspect(|ty| {
                         ender |= matches!(ty, TestEventType::Complete { .. });
                     }));
                 }
+
+                println!();
             }
         }
     }
@@ -160,5 +162,19 @@ impl TestHandler for UserHandler {
 impl Drop for UserHandler {
     fn drop(&mut self) {
         assert!(self.cache.is_empty());
+
+        // Write the final bit to the stdout.
+        let mut stdout = io::stdout().lock();
+
+        if !self.failures.is_empty() {
+            writeln!(stdout, "Test Failures:").ok();
+
+            for (test_name, panic) in &self.failures {
+                writeln!(stdout, "  {}", test_name).ok();
+                writeln!(stdout, "-------------").ok();
+                writeln!(stdout, "{}", panic).ok();
+                writeln!(stdout, "-------------").ok();
+            }
+        }
     }
 }