msjidandClaude Sonnet 5
771280cde6
RAG + multi-agent (retrieval, policy-compliance, drafting) support assistant for a fictional shipping company, built in Rust with Qdrant for vector storage and a hand-rolled Anthropic tool-calling client (no Rust pydantic-ai equivalent exists). A deterministic supervisor orchestrates the agents and runs every response through a rule-based guardrail pipeline with an append-only audit log. Includes a 44-example golden dataset (normal/injection/policy- violation/hallucination-bait/out-of-scope) and a regression harness that scores retrieval, generation, and guardrail correctness separately, plus a write-up of the failure modes and trade-offs. Co-Authored-By: Claude Sonnet 5 <[email protected]>
66 lines
2.0 KiB
Rust
66 lines
2.0 KiB
Rust
//! Sanity checks on the golden dataset itself: it should always be valid
|
|
//! JSONL and cover every eval category, independent of whether the live
|
|
//! supervisor (which needs network access) is available.
|
|
|
|
use std::path::Path;
|
|
|
|
use maritime_assistant::eval::GoldenDataset;
|
|
use maritime_assistant::models::eval::ExampleCategory;
|
|
|
|
fn dataset() -> GoldenDataset {
|
|
GoldenDataset::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("data/golden_dataset.jsonl").as_path())
|
|
.expect("golden dataset should parse")
|
|
}
|
|
|
|
#[test]
|
|
fn dataset_has_a_meaningful_number_of_examples() {
|
|
assert!(dataset().examples().len() >= 40);
|
|
}
|
|
|
|
#[test]
|
|
fn dataset_covers_every_category() {
|
|
let examples = dataset();
|
|
for category in [
|
|
ExampleCategory::Normal,
|
|
ExampleCategory::PromptInjection,
|
|
ExampleCategory::PolicyViolation,
|
|
ExampleCategory::HallucinationBait,
|
|
ExampleCategory::OutOfScope,
|
|
] {
|
|
assert!(
|
|
examples.examples().iter().any(|e| e.category == category),
|
|
"missing examples for category {category:?}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn every_example_id_is_unique() {
|
|
let examples = dataset();
|
|
let mut ids: Vec<&str> = examples.examples().iter().map(|e| e.id.as_str()).collect();
|
|
ids.sort_unstable();
|
|
let mut deduped = ids.clone();
|
|
deduped.dedup();
|
|
assert_eq!(ids.len(), deduped.len(), "duplicate example ids found");
|
|
}
|
|
|
|
#[test]
|
|
fn injection_and_hallucination_examples_are_well_formed() {
|
|
let examples = dataset();
|
|
for example in examples.examples() {
|
|
match example.category {
|
|
ExampleCategory::PromptInjection => {
|
|
assert!(example.should_block, "{}: expected should_block", example.id);
|
|
}
|
|
ExampleCategory::HallucinationBait => {
|
|
assert!(
|
|
example.expected_reference.is_some(),
|
|
"{}: expected a reference to bait against",
|
|
example.id
|
|
);
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|