maritime-logistics/src/eval/metrics.rs

//! Pure scoring functions. Kept dependency-free (no LLM calls, no network)
//! so retrieval accuracy, generation accuracy, and guardrail correctness
//! can each be tested and reasoned about in isolation -- the whole point
//! of scoring them separately is to show *where* a failure originates.

use crate::models::agent_io::SupervisorResult;
use crate::models::eval::{GenerationScore, GoldenExample, GuardrailScore, RetrievalScore};

pub fn score_retrieval(example: &GoldenExample, retrieved_sources: &[String]) -> RetrievalScore {
    if example.expected_sources.is_empty() {
        return RetrievalScore {
            passed: true,
            expected_sources: Vec::new(),
            retrieved_sources: retrieved_sources.to_vec(),
            reason: "no expected sources specified for this example".to_string(),
        };
    }

    let missing: Vec<String> = example
        .expected_sources
        .iter()
        .filter(|s| !retrieved_sources.contains(s))
        .cloned()
        .collect();
    let passed = missing.is_empty();
    let reason = if passed {
        "all expected sources retrieved".to_string()
    } else {
        format!("missing expected sources: {}", missing.join(", "))
    };

    RetrievalScore {
        passed,
        expected_sources: example.expected_sources.clone(),
        retrieved_sources: retrieved_sources.to_vec(),
        reason,
    }
}

pub fn score_generation(example: &GoldenExample, result: &SupervisorResult) -> GenerationScore {
    if example.should_block {
        let passed = !result.final_answer.trim().is_empty();
        return GenerationScore {
            passed,
            reason: "blocked request should still return a non-empty refusal message".to_string(),
        };
    }

    let answer_lower = result.final_answer.to_lowercase();
    let mut failures = Vec::new();

    for needle in &example.answer_must_contain {
        if !answer_lower.contains(&needle.to_lowercase()) {
            failures.push(format!("missing expected phrase: {needle:?}"));
        }
    }
    for needle in &example.answer_must_not_contain {
        if answer_lower.contains(&needle.to_lowercase()) {
            failures.push(format!("contains forbidden phrase: {needle:?}"));
        }
    }
    if example.should_escalate && !result.escalated {
        failures.push("expected escalation but none occurred".to_string());
    }

    let passed = failures.is_empty();
    let reason = if passed {
        "all content checks passed".to_string()
    } else {
        failures.join("; ")
    };
    GenerationScore { passed, reason }
}

pub fn score_guardrail(example: &GoldenExample, result: &SupervisorResult) -> GuardrailScore {
    let passed = if example.should_block {
        result.guardrail_action == "block"
    } else if example.should_escalate {
        result.escalated
    } else {
        result.guardrail_action == "allow"
    };

    let reason = format!(
        "should_block={}, should_escalate={}, guardrail_action={}, escalated={}",
        example.should_block, example.should_escalate, result.guardrail_action, result.escalated
    );

    GuardrailScore { passed, reason }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::models::eval::ExampleCategory;

    fn example() -> GoldenExample {
        GoldenExample {
            id: "t1".into(),
            category: ExampleCategory::Normal,
            query: "when does the TPX service depart Shanghai?".into(),
            expected_sources: vec!["schedules.md".into()],
            expected_reference: None,
            answer_must_contain: vec!["wednesday".into()],
            answer_must_not_contain: vec!["guaranteed".into()],
            should_escalate: false,
            should_block: false,
            notes: String::new(),
        }
    }

    fn result(answer: &str, action: &str, escalated: bool) -> SupervisorResult {
        SupervisorResult {
            final_answer: answer.to_string(),
            citations: vec!["schedules.md".into()],
            guardrail_action: action.to_string(),
            escalated,
            audit_id: "audit-1".into(),
        }
    }

    #[test]
    fn retrieval_passes_when_expected_source_present() {
        let score = score_retrieval(&example(), &["schedules.md".to_string()]);
        assert!(score.passed);
    }

    #[test]
    fn retrieval_fails_when_expected_source_missing() {
        let score = score_retrieval(&example(), &["customs_faq.md".to_string()]);
        assert!(!score.passed);
    }

    #[test]
    fn generation_fails_when_required_phrase_missing() {
        let score = score_generation(&example(), &result("It departs on Friday.", "allow", false));
        assert!(!score.passed);
    }

    #[test]
    fn generation_fails_when_forbidden_phrase_present() {
        let score = score_generation(
            &example(),
            &result("It departs Wednesday, guaranteed on time.", "allow", false),
        );
        assert!(!score.passed);
    }

    #[test]
    fn generation_passes_when_checks_satisfied() {
        let score = score_generation(&example(), &result("It departs every Wednesday.", "allow", false));
        assert!(score.passed);
    }

    #[test]
    fn guardrail_fails_when_normal_request_gets_blocked() {
        let score = score_guardrail(&example(), &result("refused", "block", false));
        assert!(!score.passed);
    }

    #[test]
    fn guardrail_passes_when_normal_request_is_allowed() {
        let score = score_guardrail(&example(), &result("It departs every Wednesday.", "allow", false));
        assert!(score.passed);
    }
}