From 1bdba3cacc104d0ba64e8cf4aeea1ea18107f742 Mon Sep 17 00:00:00 2001 From: John Nunley Date: Sun, 3 Mar 2024 20:19:22 -0800 Subject: [PATCH] it: Create the 'gui-test' crate The 'gui-test' crate is intended to provide a test framework for process isolated and remote test cases. Like how I intend to test winit. Signed-off-by: John Nunley --- Cargo.toml | 1 + it/gui-test/Cargo.toml | 15 ++ it/gui-test/src/lib.rs | 391 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 407 insertions(+) create mode 100644 it/gui-test/Cargo.toml create mode 100644 it/gui-test/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index e90373537..ae8dd429d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -264,6 +264,7 @@ name = "window" resolver = "2" members = [ "dpi", + "it/gui-test", "run-wasm", ] diff --git a/it/gui-test/Cargo.toml b/it/gui-test/Cargo.toml new file mode 100644 index 000000000..c83072fc2 --- /dev/null +++ b/it/gui-test/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "gui-test" +version = "0.1.0" +rust-version.workspace = true +repository.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +async-executor = "1.8.0" +async-io = "2.3.1" +async-lock = "3.3.0" +async-process = "2.1.0" +inventory = "0.3.15" +serde = { workspace = true, features = ["derive"] } diff --git a/it/gui-test/src/lib.rs b/it/gui-test/src/lib.rs new file mode 100644 index 000000000..4bcc956cd --- /dev/null +++ b/it/gui-test/src/lib.rs @@ -0,0 +1,391 @@ +//! A testing framework that can be run remotely. + +use serde::{Deserialize, Serialize}; + +use std::env; +use std::ffi::{OsStr, OsString}; +use std::mem; +use std::num::NonZeroUsize; + +const GUI_TEST_CURRENT_TEST_NAME: &str = "GUI_TEST_CURRENT_TEST_NAME"; +const GUI_TEST_SUBPROCESS_LIMIT: &str = "GUI_TEST_SUBPROCESS_LIMIT"; +const DEFAULT_LIMIT: usize = 4; + +#[doc(hidden)] +pub use inventory as __inventory; + +/// Replacement for the `main` function. +#[macro_export] +macro_rules! main { + ($harness:expr) => { + fn main() { + $crate::__entry($harness) + } + }; +} + +/// Set up a test for the test framework. +#[macro_export] +macro_rules! test { + ( + $(#[$attr:meta])* + fn $name:ident ($hname:ident : $htype:ty) $bl:block + ) => { + const _: () = { + $(#[$attr])* + fn $name ($hname: $htype) $bl + + $crate::__inventory::submit! { + $crate::__TestStart::__new( + stringify!($name), + $name + ) + } + }; + }; +} + +/// Test start. +#[doc(hidden)] +pub struct __TestStart { + /// The name of the test. + name: &'static str, + + /// The function to call. + func: fn(&mut Harness), +} + +impl __TestStart { + /// Create a new test start. + #[doc(hidden)] + pub fn __new(name: &'static str, func: fn(&mut Harness)) -> Self { + Self { name, func } + } +} + +inventory::collect! { + __TestStart +} + +/// A harness for running the tests. +pub struct Harness { + /// Name of the test start. + name: String, + + /// The inner test handler. + handler: Box, + + /// Number of tests that have been run so far. + test_count: usize, + + /// Number of tests that have failed so far. + test_fails: usize, + + /// Number of tests that have succeeded. + test_passed: usize, + + /// Current state of the test harness. + state: State, +} + +impl Harness { + /// Create a new test harness. + fn new(name: &str, handler: H) -> Self { + Self { + name: name.to_string(), + handler: Box::new(handler), + test_count: 0, + test_fails: 0, + test_passed: 0, + state: State::Default, + } + } + + /// Begin a test. + pub fn test(&mut self, name: impl Into) -> Testing<'_> { + // Make sure we aren't mid test. + match mem::replace(&mut self.state, State::Default) { + State::InTest { past_groups } => { + self.state = State::InTest { past_groups }; + panic!("tried to start a test while another was underway"); + } + + State::InGroups(groups) => { + self.state = State::InTest { + past_groups: Some(groups), + }; + } + + State::Default => { + self.state = State::InTest { past_groups: None }; + } + } + + // Send the "test started" event to the handler. + self.handler.handle_test(TestEvent { + runner: self.name.clone(), + ty: TestEventType::TestStarted { name: name.into() }, + }); + + // Return the handle. + Testing { + harness: Some(self), + } + } + + /// Run a closure as a test. + pub fn with_test(&mut self, name: impl Into, f: impl FnOnce() -> T) -> T { + let _test = self.test(name.into()); + f() + } + + /// Begin a test group. + pub fn group(&mut self, name: impl Into) -> Grouping<'_> { + // Make sure we can begin a group. + match mem::replace(&mut self.state, State::Default) { + State::Default => { + self.state = State::InGroups(NonZeroUsize::new(1).unwrap()); + } + State::InGroups(groups) => { + self.state = State::InGroups(groups.checked_add(1).unwrap()); + } + State::InTest { past_groups } => { + self.state = State::InTest { past_groups }; + panic!("cannot start group mid-test") + } + } + + // Send the "group started" event to the handler. + self.handler.handle_test(TestEvent { + ty: TestEventType::GroupStarted { name: name.into() }, + runner: self.name.clone(), + }); + + // Return the handle. + Grouping { harness: self } + } + + /// Run a closure inside of a group. + pub fn with_group(&mut self, name: impl Into, f: impl FnOnce(&mut Self) -> T) -> T { + let mut group = self.group(name.into()); + f(group.harness()) + } + + /// End an ongoing test. + fn end_test(&mut self, reason: TestResult) { + self.test_count += 1; + match &reason { + TestResult::Passed => self.test_passed += 1, + TestResult::Failed(..) => self.test_fails += 1, + _ => {} + } + + self.handler.handle_test(TestEvent { + runner: self.name.clone(), + ty: TestEventType::TestEnded { result: reason }, + }); + + let count = match mem::replace(&mut self.state, State::Default) { + State::InTest { past_groups } => past_groups, + _ => unreachable!(), + }; + + self.state = match count { + None => State::Default, + Some(count) => State::InGroups(count), + }; + } + + /// End the current group. + fn end_group(&mut self) { + self.handler.handle_test(TestEvent { + runner: self.name.clone(), + ty: TestEventType::GroupEnded, + }); + + let count = match mem::replace(&mut self.state, State::Default) { + State::InGroups(groups) => groups, + _ => unreachable!(), + }; + + self.state = match NonZeroUsize::new(count.get() - 1) { + None => State::Default, + Some(groups) => State::InGroups(groups), + }; + } +} + +/// An in-progress test. +pub struct Testing<'a> { + harness: Option<&'a mut Harness>, +} + +impl Testing<'_> { + /// Skip this test. + pub fn skip(mut self) { + // Send the "skipped" event. + self.harness.take().unwrap().end_test(TestResult::Skipped); + } +} + +impl Drop for Testing<'_> { + fn drop(&mut self) { + if let Some(harness) = self.harness.take() { + let result = if std::thread::panicking() { + TestResult::Failed("thread panicked".into()) + } else { + TestResult::Passed + }; + + harness.end_test(result); + } + } +} + +/// We are running a test group. +pub struct Grouping<'a> { + harness: &'a mut Harness, +} + +impl Grouping<'_> { + /// Get the underlying test harness. + pub fn harness(&mut self) -> &mut Harness { + &mut self.harness + } +} + +impl Drop for Grouping<'_> { + fn drop(&mut self) { + self.harness.end_group(); + } +} + +/// Current testing state. +enum State { + /// We are in the middle of this many groups. + InGroups(NonZeroUsize), + + /// We are in the middle of a test. + InTest { past_groups: Option }, + + /// We are in the default state. + Default, +} + +/// A handler for incoming test events. +pub trait TestHandler { + /// Handle a test. + fn handle_test(&mut self, event: TestEvent); +} + +/// An event produced by the test harness. +#[non_exhaustive] +#[derive(Debug, Serialize, Deserialize)] +pub struct TestEvent { + /// The name of the runner associated with the event. + runner: String, + + /// The type of the event. + ty: TestEventType, +} + +/// The type of the event. +#[non_exhaustive] +#[derive(Debug, Serialize, Deserialize)] +pub enum TestEventType { + /// The tests are complete and the harness can be disconnected. + Complete, + + /// A test has started. + TestStarted { name: String }, + + /// A test has completed. + TestEnded { result: TestResult }, + + /// A test group has started. + GroupStarted { name: String }, + + /// A test group has ended. + GroupEnded, +} + +/// The result of a test. +#[non_exhaustive] +#[derive(Debug, Serialize, Deserialize)] +pub enum TestResult { + /// The test passed. + Passed, + + /// The test failed with the provided error. + Failed(String), + + /// The test was skipped. + Skipped, +} + +/// Entry point of the test. +#[doc(hidden)] +pub fn __entry(handler: H) { + // Look for the test name environment variable. + if let Some(test_name) = env::var(GUI_TEST_CURRENT_TEST_NAME) + .ok() + .filter(|test_name| !test_name.is_empty()) + { + // Find the provided test. + let test_to_run = inventory::iter::<__TestStart> + .into_iter() + .find(|test| test.name == test_name) + .unwrap_or_else(|| panic!("unable to find test '{test_name}'")); + + // Create a harness. + let mut harness = Harness::new(test_to_run.name, handler); + + // Run the test. + (test_to_run.func)(&mut harness); + } else { + // Run a subprocess for every test. + let limit = env::var(GUI_TEST_SUBPROCESS_LIMIT) + .ok() + .and_then(|limit| limit.parse::().ok()) + .unwrap_or(DEFAULT_LIMIT); + let process_name = env::args_os().next().unwrap(); + + let sema = async_lock::Semaphore::new(limit); + let ex = async_executor::Executor::new(); + + async_io::block_on(ex.run(async { + let mut tasks = vec![]; + + // Set up an environment variable for this. + for test in inventory::iter::<__TestStart> { + // Acquire a guard. + let guard = sema.acquire().await; + + // Spawn a subprocess. + let mut process = async_process::Command::new(&process_name) + .envs(env::vars_os().chain(Some({ + (path(&GUI_TEST_CURRENT_TEST_NAME), path(&test.name)) + }))) + .spawn() + .expect("failed to spawn child process"); + + // Spawn a task to poll that subprocess. + let task = ex.spawn(async move { + let _guard = guard; + process.status().await.unwrap() + }); + + tasks.push(task); + } + + // Finish all of the tasks. + for task in tasks { + task.await; + } + })); + } +} + +fn path>(s: &A) -> OsString { + s.as_ref().into() +}