feat: provide much better debugging experience

Author: ljahier

src/cli/commands.rs

@@ -33,6 +33,9 @@ pub enum Commands {
             help = "Set variables in key=value format (comma-separated: key1=val1,key2=val2)"
         )]
         var: Option<String>,
+
+        #[arg(long, default_value = "false", help = "Disable failure summary display")]
+        no_fail_summary: bool,
     },
 
     Validate {

src/cli/runner.rs

@@ -16,6 +16,7 @@ pub fn run(opts: Opts) {
             file,
             debug: debug_enabled,
             var,
+            no_fail_summary,
         } => {
             if debug_enabled {
                 debug::enable_debug();
@@ -27,6 +28,7 @@ pub fn run(opts: Opts) {
                 disable_color,
                 file,
                 var,
+                no_fail_summary,
             ));
         }
         Commands::Validate { file, var } => {
@@ -54,7 +56,8 @@ pub async fn run_tests(
     disable_color: bool,
     file: Option<String>,
     var: Option<String>,
+    no_fail_summary: bool,
 ) {
     let mut runner = TestRunner::new(disable_color);
-    runner.execute_tests(filter, verbose, file, var).await;
+    runner.execute_tests(filter, verbose, file, var, no_fail_summary).await;
 }

src/core/runner.rs

@@ -25,6 +25,8 @@ pub struct TestResult {
     pub response_body: Option<Value>,
     pub headers: HashMap<String, String>,
     pub messages: Vec<String>,
+    pub method: String,
+    pub endpoint: String,
 }
 
 struct TestExecutionContext<'a> {
@@ -40,6 +42,7 @@ pub struct TestRunner {
     pub variables: HashMap<String, String>,
     pub results: Vec<TestResult>,
     pub disable_color: bool,
+    pub no_fail_summary: bool,
 }
 
 impl TestRunner {
@@ -49,6 +52,7 @@ impl TestRunner {
             variables: HashMap::new(),
             results: Vec::new(),
             disable_color,
+            no_fail_summary: false,
         }
     }
 
@@ -73,6 +77,8 @@ impl TestRunner {
             response_body: result.body,
             headers: result.headers,
             messages: result.errors,
+            method: test.method.clone(),
+            endpoint: test.endpoint.clone(),
         }
     }
 
@@ -109,6 +115,8 @@ impl TestRunner {
                 response_body: None,
                 headers: HashMap::new(),
                 messages: error_messages.clone(),
+                method: test.method.clone(),
+                endpoint: test.endpoint.clone(),
             }
         } else {
             self.execute_test(test, client, test_file_dir).await
@@ -149,8 +157,10 @@ impl TestRunner {
         verbose: bool,
         file: Option<String>,
         var: Option<String>,
+        no_fail_summary: bool,
     ) {
         load_env_files();
+        self.no_fail_summary = no_fail_summary;
 
         // Parse CLI variables and add them to the runner's variables
         let cli_variables = crate::cli::Commands::parse_variables(var);
@@ -339,16 +349,45 @@ impl TestRunner {
             self.results.push(result);
         }
 
-        self.display_summary(*skipped, context.total);
+        self.display_compact_results(*skipped, context.total, context.verbose);
+        if !self.no_fail_summary && !context.verbose {
+            self.display_failure_details();
+        }
     }
 
-    fn display_summary(&self, skipped: usize, total: usize) {
+    fn display_compact_results(&self, skipped: usize, total: usize, verbose: bool) {
+        // Only display compact results in non-verbose mode
+        if verbose {
+            return;
+        }
+        
         if !self.disable_color {
             println!("\n{}", "━".repeat(get_terminal_width()).blue());
+            println!("\nResults:");
+            
+            for result in &self.results {
+                let status_indicator = if result.success {
+                    "✓".green()
+                } else {
+                    "✗".red()
+                };
+                
+                if result.success {
+                    println!("  {} {}", status_indicator, result.name);
+                } else {
+                    println!("  {} {} (expected {}, got {})", 
+                        status_indicator, 
+                        result.name,
+                        result.expected_status,
+                        result.actual_status.to_string().red()
+                    );
+                }
+            }
+            
+            println!();
             let success_count = self.results.iter().filter(|r| r.success).count();
             let fail_count = self.results.len() - success_count;
 
-            println!("\nSummary:");
             if success_count > 0 {
                 print!("{} passed", format!("{success_count} tests").green());
             }
@@ -365,6 +404,103 @@ impl TestRunner {
                 print!("{} skipped", format!("{skipped} tests").yellow());
             }
             println!(" (total: {total})");
+        } else {
+            println!("\nResults:");
+            for result in &self.results {
+                if result.success {
+                    println!("  ✓ {}", result.name);
+                } else {
+                    println!("  ✗ {} (expected {}, got {})", 
+                        result.name, result.expected_status, result.actual_status);
+                }
+            }
+            
+            let success_count = self.results.iter().filter(|r| r.success).count();
+            let fail_count = self.results.len() - success_count;
+            
+            print!("\n{success_count} tests passed");
+            if fail_count > 0 {
+                print!(", {fail_count} tests failed");
+            }
+            if skipped > 0 {
+                print!(", {skipped} tests skipped");
+            }
+            println!(" (total: {total})");
+        }
+    }
+
+    fn display_failure_details(&self) {
+        let failed_results: Vec<_> = self.results.iter().filter(|r| !r.success).collect();
+        
+        if failed_results.is_empty() {
+            return;
+        }
+        
+        if !self.disable_color {
+            println!("\n{}", "━".repeat(get_terminal_width()).blue());
+            println!("\n{}", "Failures:".red().bold());
+        } else {
+            println!("\nFailures:");
+        }
+        
+        for (i, result) in failed_results.iter().enumerate() {
+            if i > 0 {
+                println!();
+            }
+            
+            if !self.disable_color {
+                println!("---");
+                println!("Test: {}", result.name.bold());
+                println!("Endpoint: {} {}", 
+                    result.method.yellow(), 
+                    result.endpoint.yellow()
+                );
+                println!("Status: {} (expected {})", 
+                    result.actual_status.to_string().red(),
+                    result.expected_status.to_string().bold()
+                );
+            } else {
+                println!("---");
+                println!("Test: {}", result.name);
+                println!("Endpoint: {} {}", result.method, result.endpoint);
+                println!("Status: {} (expected {})", result.actual_status, result.expected_status);
+            }
+            
+            if !result.messages.is_empty() {
+                println!("Messages:");
+                for msg in &result.messages {
+                    if !self.disable_color {
+                        println!("  - {}", msg.red());
+                    } else {
+                        println!("  - {msg}");
+                    }
+                }
+            }
+            
+            if let Some(body) = &result.response_body {
+                println!("Response Body:");
+                let body_str = self.format_response_body(body);
+                println!("{body_str}");
+            } else {
+                println!("Response Body: <none>");
+            }
+        }
+    }
+    
+    pub const MAX_SUMMARY_BODY_BYTES: usize = 8192;
+    
+    pub fn format_response_body(&self, body: &Value) -> String {
+        let formatted = if body.is_object() || body.is_array() {
+            serde_json::to_string_pretty(body).unwrap_or_else(|_| body.to_string())
+        } else {
+            body.to_string()
+        };
+        
+        if formatted.len() > Self::MAX_SUMMARY_BODY_BYTES {
+            let truncated = &formatted[..Self::MAX_SUMMARY_BODY_BYTES];
+            format!("{truncated}\n... (truncated)")
+        } else {
+            formatted
         }
     }
 }

tests/failure_summary_tests.rs

@@ -0,0 +1,137 @@
+use std::collections::HashMap;
+use serde_json::json;
+use catalyst::core::runner::{TestRunner, TestResult};
+
+#[tokio::test]
+async fn test_failure_summary_display() {
+    let mut runner = TestRunner::new(false); // colored output enabled
+    runner.no_fail_summary = false; // failure summary enabled
+    
+    // Add some test results with failures
+    runner.results.push(TestResult {
+        name: "Test 1 - Success".to_string(),
+        success: true,
+        expected_status: 200,
+        actual_status: 200,
+        response_body: Some(json!({"message": "success"})),
+        headers: HashMap::new(),
+        messages: vec![],
+        method: "GET".to_string(),
+        endpoint: "/api/users".to_string(),
+    });
+    
+    runner.results.push(TestResult {
+        name: "Test 2 - Failure".to_string(),
+        success: false,
+        expected_status: 200,
+        actual_status: 404,
+        response_body: Some(json!({"error": "Not found", "code": 404})),
+        headers: HashMap::new(),
+        messages: vec!["Status mismatch".to_string(), "Body validation failed".to_string()],
+        method: "POST".to_string(),
+        endpoint: "/api/users/123".to_string(),
+    });
+    
+    runner.results.push(TestResult {
+        name: "Test 3 - Success".to_string(),
+        success: true,
+        expected_status: 201,
+        actual_status: 201,
+        response_body: Some(json!({"id": 123, "created": true})),
+        headers: HashMap::new(),
+        messages: vec![],
+        method: "POST".to_string(),
+        endpoint: "/api/users".to_string(),
+    });
+    
+    // Verify that we have the expected mix of results
+    let success_count = runner.results.iter().filter(|r| r.success).count();
+    let fail_count = runner.results.iter().filter(|r| !r.success).count();
+    
+    assert_eq!(success_count, 2);
+    assert_eq!(fail_count, 1);
+    
+    // Test the format_response_body method
+    let json_body = json!({"error": "Not found", "code": 404});
+    let formatted = runner.format_response_body(&json_body);
+    assert!(formatted.contains("\"error\": \"Not found\""));
+    assert!(formatted.contains("\"code\": 404"));
+    
+    // Test truncation with large body
+    let large_json = json!({
+        "data": "x".repeat(10000),
+        "message": "This is a large response"
+    });
+    let truncated = runner.format_response_body(&large_json);
+    assert!(truncated.contains("... (truncated)"));
+    assert!(truncated.len() <= TestRunner::MAX_SUMMARY_BODY_BYTES + 20); // Allow for truncation message
+}
+
+#[tokio::test]
+async fn test_no_fail_summary_flag() {
+    let mut runner = TestRunner::new(false);
+    runner.no_fail_summary = true; // failure summary disabled
+    
+    // Add a failing test result
+    runner.results.push(TestResult {
+        name: "Failing Test".to_string(),
+        success: false,
+        expected_status: 200,
+        actual_status: 500,
+        response_body: Some(json!({"error": "Internal server error"})),
+        headers: HashMap::new(),
+        messages: vec!["Server error occurred".to_string()],
+        method: "GET".to_string(),
+        endpoint: "/api/status".to_string(),
+    });
+    
+    // When no_fail_summary is true, failure details should not be displayed
+    // This is tested indirectly by checking the flag value
+    assert!(runner.no_fail_summary);
+}
+
+#[tokio::test]
+async fn test_body_truncation_logic() {
+    let runner = TestRunner::new(false);
+    
+    // Test normal sized body
+    let normal_body = json!({"message": "Hello world"});
+    let formatted_normal = runner.format_response_body(&normal_body);
+    assert!(!formatted_normal.contains("... (truncated)"));
+    
+    // Test large body that exceeds MAX_SUMMARY_BODY_BYTES
+    let large_data = "x".repeat(TestRunner::MAX_SUMMARY_BODY_BYTES + 1000);
+    let large_body = json!({"data": large_data});
+    let formatted_large = runner.format_response_body(&large_body);
+    assert!(formatted_large.contains("... (truncated)"));
+    assert!(formatted_large.len() <= TestRunner::MAX_SUMMARY_BODY_BYTES + 50);
+    
+    // Test non-JSON body (string)
+    let string_body = json!("This is just a string response");
+    let formatted_string = runner.format_response_body(&string_body);
+    assert_eq!(formatted_string, "\"This is just a string response\"");
+}
+
+#[tokio::test]
+async fn test_test_result_structure() {
+    // Test that TestResult properly captures method and endpoint
+    let result = TestResult {
+        name: "Test API call".to_string(),
+        success: true,
+        expected_status: 200,
+        actual_status: 200,
+        response_body: Some(json!({"status": "ok"})),
+        headers: HashMap::new(),
+        messages: vec![],
+        method: "GET".to_string(),
+        endpoint: "/api/status".to_string(),
+    };
+    
+    assert_eq!(result.name, "Test API call");
+    assert!(result.success);
+    assert_eq!(result.expected_status, 200);
+    assert_eq!(result.actual_status, 200);
+    assert_eq!(result.method, "GET");
+    assert_eq!(result.endpoint, "/api/status");
+    assert!(result.response_body.is_some());
+}