From dfa8e8439f66a763a352d32f9b153a81466ea2cf Mon Sep 17 00:00:00 2001 From: Adam Petro Date: Thu, 10 Oct 2024 17:08:24 -0400 Subject: [PATCH] WIP --- Cargo.lock | 7 ++ Cargo.toml | 1 + src/lib.rs | 1 + src/main.rs | 209 ++++++++++++++++++++++++++++++++++++--------- src/test_report.rs | 84 ++++++++++++++++++ 5 files changed, 264 insertions(+), 38 deletions(-) create mode 100644 src/test_report.rs diff --git a/Cargo.lock b/Cargo.lock index af542b66..0512fd01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -977,6 +977,7 @@ dependencies = [ "rust-embed", "serde", "serde_json", + "similar", "wasi-common", "wasmprof", "wasmtime 22.0.0", @@ -1967,6 +1968,12 @@ dependencies = [ "dirs", ] +[[package]] +name = "similar" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e" + [[package]] name = "slice-group-by" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index 4ba252be..41f44028 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ wasmprof = "0.7.0" bluejay-core = { version = "=0.2.0" } bluejay-parser = { version = "=0.2.0", features = ["format-errors"] } bluejay-validator = { version = "=0.2.0" } +similar = "2.6.0" [dev-dependencies] assert_cmd = "2.0" diff --git a/src/lib.rs b/src/lib.rs index f4bf5142..d274b188 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,3 +3,4 @@ pub mod engine; pub mod function_run_result; pub mod logs; pub mod scale_limits_analyzer; +pub mod test_report; diff --git a/src/main.rs b/src/main.rs index f3be57e8..d5771678 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,10 +5,13 @@ use std::{ }; use anyhow::{anyhow, Result}; -use clap::{Parser, ValueEnum}; +use clap::{Args, Parser, Subcommand, ValueEnum}; +use colored::Colorize; use function_runner::{ bluejay_schema_analyzer::BluejaySchemaAnalyzer, - engine::{run, FunctionRunParams, ProfileOpts}, + engine::{run as engine_run, FunctionRunParams, ProfileOpts}, + function_run_result::FunctionOutput, + test_report::TestReport, }; use is_terminal::IsTerminal; @@ -27,27 +30,66 @@ enum Codec { JsonToMessagepack, } -/// Simple Function runner which takes JSON as a convenience. #[derive(Parser, Debug)] #[clap(version)] #[command(arg_required_else_help = true)] -struct Opts { +struct Cli { + #[clap(subcommand)] + cmd: Command, +} + +#[derive(Subcommand, Debug)] +enum Command { + Run(RunOpts), + Test(TestOpts), +} + +#[derive(Args, Debug)] +struct CommonOpts { /// Path to wasm/wat Function #[clap(short, long, default_value = "function.wasm")] function: PathBuf, - /// Path to json file containing Function input; if omitted, stdin is used - #[clap(short, long)] - input: Option, - /// Name of the export to invoke. #[clap(short, long, default_value = "_start")] export: String, - /// Log the run result as a JSON object + /// Log the result as a JSON object #[clap(short, long)] json: bool, + #[clap(short = 'c', long, value_enum, default_value = "json")] + codec: Codec, + + /// Path to graphql file containing Function schema; if omitted, defaults will be used to calculate limits. + #[clap(short = 's', long)] + schema_path: Option, + + /// Path to graphql file containing Function input query; if omitted, defaults will be used to calculate limits. + #[clap(short = 'q', long)] + query_path: Option, +} + +impl CommonOpts { + pub fn read_schema_to_string(&self) -> Option> { + self.schema_path.as_ref().map(read_file_to_string) + } + + pub fn read_query_to_string(&self) -> Option> { + self.query_path.as_ref().map(read_file_to_string) + } +} + +/// Simple Function runner which takes JSON as a convenience. +#[derive(Args, Debug)] +struct RunOpts { + #[clap(flatten)] + common_opts: CommonOpts, + + /// Path to json file containing Function input; if omitted, stdin is used + #[clap(short, long)] + input: Option, + /// Enable profiling. This will make your Function run slower. /// The resulting profile can be used in speedscope (https://www.speedscope.app/) /// Specifying --profile-* argument will also enable profiling. @@ -61,20 +103,9 @@ struct Opts { /// How many samples per second. Defaults to 500_000 (every 2us). #[clap(long)] profile_frequency: Option, - - #[clap(short = 'c', long, value_enum, default_value = "json")] - codec: Codec, - - /// Path to graphql file containing Function schema; if omitted, defaults will be used to calculate limits. - #[clap(short = 's', long)] - schema_path: Option, - - /// Path to graphql file containing Function input query; if omitted, defaults will be used to calculate limits. - #[clap(short = 'q', long)] - query_path: Option, } -impl Opts { +impl RunOpts { pub fn profile_opts(&self) -> Option { if !self.profile && self.profile_out.is_none() && self.profile_frequency.is_none() { return None; @@ -93,7 +124,8 @@ impl Opts { let mut path = PathBuf::new(); path.set_file_name( - self.function + self.common_opts + .function .file_name() .unwrap_or(std::ffi::OsStr::new("function")), ); @@ -101,14 +133,20 @@ impl Opts { path } +} - pub fn read_schema_to_string(&self) -> Option> { - self.schema_path.as_ref().map(read_file_to_string) - } +#[derive(Args, Debug)] +struct TestOpts { + #[clap(flatten)] + common_opts: CommonOpts, - pub fn read_query_to_string(&self) -> Option> { - self.query_path.as_ref().map(read_file_to_string) - } + /// Path to directory containing Function input test cases + #[clap(short, long, default_value = "integration_tests/input/")] + input: PathBuf, + + /// Path to directory containing Function test case expected outputs + #[clap(short, long, default_value = "integration_tests/output/")] + output: PathBuf, } fn read_file_to_string(file_path: &PathBuf) -> Result { @@ -123,8 +161,15 @@ fn read_file_to_string(file_path: &PathBuf) -> Result { } fn main() -> Result<()> { - let opts: Opts = Opts::parse(); + let Cli { cmd } = Cli::parse(); + match cmd { + Command::Run(opts) => run(opts), + Command::Test(opts) => test(opts), + } +} + +fn run(opts: RunOpts) -> Result<()> { let mut input: Box = if let Some(ref input) = opts.input { Box::new(BufReader::new(File::open(input).map_err(|e| { anyhow!("Couldn't load input {:?}: {}", input, e) @@ -140,11 +185,11 @@ fn main() -> Result<()> { let mut buffer = Vec::new(); input.read_to_end(&mut buffer)?; - let schema_string = opts.read_schema_to_string().transpose()?; + let schema_string = opts.common_opts.read_schema_to_string().transpose()?; - let query_string = opts.read_query_to_string().transpose()?; + let query_string = opts.common_opts.read_query_to_string().transpose()?; - let (json_value, buffer) = match opts.codec { + let (json_value, buffer) = match opts.common_opts.codec { Codec::Json => { let json = serde_json::from_slice::(&buffer) .map_err(|e| anyhow!("Invalid input JSON: {}", e))?; @@ -167,9 +212,15 @@ fn main() -> Result<()> { { BluejaySchemaAnalyzer::analyze_schema_definition( &schema_string, - opts.schema_path.as_ref().and_then(|p| p.to_str()), + opts.common_opts + .schema_path + .as_ref() + .and_then(|p| p.to_str()), &query_string, - opts.query_path.as_ref().and_then(|p| p.to_str()), + opts.common_opts + .query_path + .as_ref() + .and_then(|p| p.to_str()), &json_value, )? } else { @@ -178,15 +229,15 @@ fn main() -> Result<()> { let profile_opts = opts.profile_opts(); - let function_run_result = run(FunctionRunParams { - function_path: opts.function, + let function_run_result = engine_run(FunctionRunParams { + function_path: opts.common_opts.function, input: buffer, - export: opts.export.as_ref(), + export: opts.common_opts.export.as_ref(), profile_opts: profile_opts.as_ref(), scale_factor, })?; - if opts.json { + if opts.common_opts.json { println!("{}", function_run_result.to_json()); } else { println!("{function_run_result}"); @@ -198,3 +249,85 @@ fn main() -> Result<()> { Ok(()) } + +fn test(opts: TestOpts) -> Result<()> { + if !opts.input.is_dir() { + return Err(anyhow!("Input path must be a directory")); + } + if !opts.output.is_dir() { + return Err(anyhow!("Output path must be a directory")); + } + + if opts.common_opts.codec != Codec::Json { + return Err(anyhow!("Only JSON codec is supported for testing")); + } + + let mut report = TestReport::default(); + + let file_name_and_input_and_output_paths = std::fs::read_dir(opts.input)? + .map(|entry| { + let entry = entry?; + + let input_path = entry.path(); + + if input_path.is_dir() { + Ok(None) + } else if let Some(file_name) = input_path.file_name() { + let file_name = file_name.to_string_lossy().into_owned(); + let output_path = opts.output.join(&file_name); + Ok(Some((file_name, input_path, output_path))) + } else { + Err(anyhow!("Invalid file name")) + } + }) + .filter_map(Result::transpose) + .collect::>>()?; + + match file_name_and_input_and_output_paths.len() { + 0 => anyhow::bail!("No test files found"), + 1 => println!("running 1 test"), + n => println!("running {n} tests"), + } + + file_name_and_input_and_output_paths + .into_iter() + .try_for_each( + |(file_name, input_path, output_path)| -> anyhow::Result<()> { + let input = read_file_to_string(&input_path)?; + let output = read_file_to_string(&output_path)?; + + let output = serde_json::from_str::(&output) + .map_err(|e| anyhow!("Invalid output JSON: {}", e))?; + + print!("test {} ...", &file_name); + + let function_run_result = engine_run(FunctionRunParams { + function_path: opts.common_opts.function.clone(), + input: input.into_bytes(), + export: opts.common_opts.export.as_ref(), + profile_opts: None, + scale_factor: 1.0, + })?; + + match &function_run_result.output { + FunctionOutput::JsonOutput(json) => { + if json != &output { + report.add_failure(file_name, output, function_run_result); + println!(" {}", "FAILED".red()); + } else { + report.add_success(); + println!(" {}", "ok".green()); + } + } + FunctionOutput::InvalidJsonOutput(_) => { + report.add_failure(file_name, output, function_run_result); + println!(" {}", "FAILED".red()); + } + } + + Ok(()) + }, + )?; + + report.into_result() +} diff --git a/src/test_report.rs b/src/test_report.rs new file mode 100644 index 00000000..01a8a1ab --- /dev/null +++ b/src/test_report.rs @@ -0,0 +1,84 @@ +use crate::function_run_result::{FunctionOutput, FunctionRunResult}; +use colored::Colorize; +use serde_json::Value; +use similar::TextDiff; + +#[derive(Default)] +pub struct TestReport { + successes: usize, + failures: Vec, +} + +impl TestReport { + pub fn add_success(&mut self) { + self.successes += 1; + } + + pub fn add_failure( + &mut self, + filename: String, + expected_output: Value, + run_result: FunctionRunResult, + ) { + self.failures.push(TestFailure { + filename, + expected_output, + run_result, + }); + } + + pub fn into_result(self) -> anyhow::Result<()> { + println!(); + + if !self.failures.is_empty() { + println!("failures:\n"); + + self.failures.iter().for_each(|failure| { + println!("{:-^40}", format!(" {} logs ", failure.filename)); + println!("{}\n", failure.run_result.logs); + println!("{:-^40}", format!(" {} output ", failure.filename)); + let output: std::borrow::Cow = match &failure.run_result.output { + FunctionOutput::JsonOutput(json) => serde_json::to_string_pretty(json) + .expect("failed to serialize JSON") + .into(), + FunctionOutput::InvalidJsonOutput(output) => (&output.stdout).into(), + }; + println!("{}\n", output.as_ref()); + + println!("{:-^40}", format!(" {} output diff ", failure.filename)); + + let expected = serde_json::to_string_pretty(&failure.expected_output) + .expect("failed to serialize JSON"); + + let diff = TextDiff::from_lines(expected.as_str(), output.as_ref()); + + println!("{}", diff.unified_diff().missing_newline_hint(false)); + + println!(); + }); + } + + println!( + "test result: {}. {} passed; {} failed", + if self.failures.is_empty() { + "ok".green() + } else { + "FAILED".red() + }, + self.successes, + self.failures.len() + ); + + if self.failures.is_empty() { + Ok(()) + } else { + Err(anyhow::anyhow!("test failed")) + } + } +} + +pub struct TestFailure { + filename: String, + expected_output: Value, + run_result: FunctionRunResult, +}