Add Maritime/Logistics support assistant with safety/eval layer
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
//! Sanity checks on the golden dataset itself: it should always be valid
|
||||
//! JSONL and cover every eval category, independent of whether the live
|
||||
//! supervisor (which needs network access) is available.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use maritime_assistant::eval::GoldenDataset;
|
||||
use maritime_assistant::models::eval::ExampleCategory;
|
||||
|
||||
fn dataset() -> GoldenDataset {
|
||||
GoldenDataset::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("data/golden_dataset.jsonl").as_path())
|
||||
.expect("golden dataset should parse")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dataset_has_a_meaningful_number_of_examples() {
|
||||
assert!(dataset().examples().len() >= 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dataset_covers_every_category() {
|
||||
let examples = dataset();
|
||||
for category in [
|
||||
ExampleCategory::Normal,
|
||||
ExampleCategory::PromptInjection,
|
||||
ExampleCategory::PolicyViolation,
|
||||
ExampleCategory::HallucinationBait,
|
||||
ExampleCategory::OutOfScope,
|
||||
] {
|
||||
assert!(
|
||||
examples.examples().iter().any(|e| e.category == category),
|
||||
"missing examples for category {category:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_example_id_is_unique() {
|
||||
let examples = dataset();
|
||||
let mut ids: Vec<&str> = examples.examples().iter().map(|e| e.id.as_str()).collect();
|
||||
ids.sort_unstable();
|
||||
let mut deduped = ids.clone();
|
||||
deduped.dedup();
|
||||
assert_eq!(ids.len(), deduped.len(), "duplicate example ids found");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn injection_and_hallucination_examples_are_well_formed() {
|
||||
let examples = dataset();
|
||||
for example in examples.examples() {
|
||||
match example.category {
|
||||
ExampleCategory::PromptInjection => {
|
||||
assert!(example.should_block, "{}: expected should_block", example.id);
|
||||
}
|
||||
ExampleCategory::HallucinationBait => {
|
||||
assert!(
|
||||
example.expected_reference.is_some(),
|
||||
"{}: expected a reference to bait against",
|
||||
example.id
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user