mirror of
https://gitea.ingwaz.work/Ingwaz/openbrain-mcp.git
synced 2026-06-15 22:07:08 +00:00
feat(truth): add scoring pipeline module (#34)
Implement truth scoring orchestrator that ties PLN + ECAN together: New types: - TruthCategory: verified/plausible/unverified/contradicted enum - ScorerConfig: pipeline configuration (thresholds, ECAN params) - RelatedMemory: cross-reference result with similarity + existing scores - ScoringResult: complete scoring output with TV, confidence, category, ECAN Core functions: - score_memory(): orchestrates evidence classification, PLN scoring, ECAN computation, and categorization - is_contradiction(): heuristic negation-asymmetry detection - categorize(): rule-based category assignment Scoring pipeline: 1. Classify related memories as confirmations or contradictions 2. Scale confidence by cosine similarity (closer = stronger evidence) 3. Apply PLN score_with_evidence for truth value computation 4. Run ECAN cycle (re-score) or initialize (first score) 5. Categorize based on TV, confirmations, and contradictions 13 unit tests covering all scoring paths, categories, contradiction detection, ECAN initialization/cycling, and output bounds. Part of #29
This commit is contained in:
@@ -13,3 +13,4 @@
|
||||
|
||||
pub mod ecan;
|
||||
pub mod pln;
|
||||
pub mod scorer;
|
||||
|
||||
396
src/truth/scorer.rs
Normal file
396
src/truth/scorer.rs
Normal file
@@ -0,0 +1,396 @@
|
||||
//! Truth scoring pipeline — orchestrates PLN, ECAN, and cross-referencing
|
||||
//! to produce truth scores for memories.
|
||||
//!
|
||||
//! This is the core intelligence module that evaluates each memory by:
|
||||
//! 1. Finding related memories via vector similarity (cross-referencing)
|
||||
//! 2. Classifying related memories as confirmations or contradictions
|
||||
//! 3. Applying PLN evidence-based scoring
|
||||
//! 4. Computing ECAN attention values (STI/LTI)
|
||||
//! 5. Categorizing the result
|
||||
|
||||
use crate::truth::ecan::EcanParams;
|
||||
use crate::truth::pln::{self, TruthValue};
|
||||
|
||||
/// Truth category assigned after scoring.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TruthCategory {
|
||||
/// TV ≥ verification_threshold AND has confirmations
|
||||
Verified,
|
||||
/// TV ≥ 0.5
|
||||
Plausible,
|
||||
/// TV < 0.5 AND no contradictions
|
||||
Unverified,
|
||||
/// Contradictions detected
|
||||
Contradicted,
|
||||
}
|
||||
|
||||
impl TruthCategory {
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
TruthCategory::Verified => "verified",
|
||||
TruthCategory::Plausible => "plausible",
|
||||
TruthCategory::Unverified => "unverified",
|
||||
TruthCategory::Contradicted => "contradicted",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for TruthCategory {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for the scoring pipeline.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScorerConfig {
|
||||
/// Base confidence for PLN scoring when no prior exists.
|
||||
pub pln_base_confidence: f32,
|
||||
/// Similarity threshold above which two memories are considered related.
|
||||
pub contradiction_threshold: f32,
|
||||
/// Truth value threshold for "verified" category.
|
||||
pub verification_threshold: f32,
|
||||
/// ECAN parameters.
|
||||
pub ecan: EcanParams,
|
||||
}
|
||||
|
||||
impl Default for ScorerConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pln_base_confidence: 0.85,
|
||||
contradiction_threshold: 0.85,
|
||||
verification_threshold: 0.8,
|
||||
ecan: EcanParams::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A related memory found during cross-referencing.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RelatedMemory {
|
||||
/// Cosine similarity to the candidate memory.
|
||||
pub similarity: f32,
|
||||
/// Content of the related memory.
|
||||
pub content: String,
|
||||
/// Existing truth value of the related memory (if scored).
|
||||
pub truth_value: Option<f32>,
|
||||
/// Existing confidence of the related memory (if scored).
|
||||
pub truth_confidence: Option<f32>,
|
||||
}
|
||||
|
||||
/// Result of scoring a single memory.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScoringResult {
|
||||
pub truth_value: f32,
|
||||
pub truth_confidence: f32,
|
||||
pub category: TruthCategory,
|
||||
pub ecan_sti: f32,
|
||||
pub ecan_lti: f32,
|
||||
pub confirmation_count: usize,
|
||||
pub contradiction_count: usize,
|
||||
}
|
||||
|
||||
/// Determine whether two pieces of content are contradictory.
|
||||
///
|
||||
/// This uses a heuristic approach: if two memories are semantically similar
|
||||
/// (high cosine similarity) but contain negation patterns, they likely
|
||||
/// contradict each other.
|
||||
///
|
||||
/// A more sophisticated implementation could use an NLI model, but this
|
||||
/// provides a reasonable baseline.
|
||||
fn is_contradiction(candidate_content: &str, related_content: &str, similarity: f32) -> bool {
|
||||
// High similarity is required — unrelated content can't contradict
|
||||
if similarity < 0.6 {
|
||||
return false;
|
||||
}
|
||||
|
||||
let negation_markers = [
|
||||
"not ", "never ", "no ", "false", "incorrect", "wrong",
|
||||
"isn't", "aren't", "wasn't", "weren't", "don't", "doesn't",
|
||||
"didn't", "won't", "wouldn't", "shouldn't", "couldn't",
|
||||
"cannot", "can't", "unlikely", "impossible", "untrue",
|
||||
];
|
||||
|
||||
let candidate_lower = candidate_content.to_lowercase();
|
||||
let related_lower = related_content.to_lowercase();
|
||||
|
||||
let candidate_negations = negation_markers
|
||||
.iter()
|
||||
.filter(|m| candidate_lower.contains(*m))
|
||||
.count();
|
||||
let related_negations = negation_markers
|
||||
.iter()
|
||||
.filter(|m| related_lower.contains(*m))
|
||||
.count();
|
||||
|
||||
// If one has significantly more negation markers than the other,
|
||||
// and they're semantically similar, it suggests contradiction.
|
||||
let negation_diff = (candidate_negations as i32 - related_negations as i32).unsigned_abs();
|
||||
|
||||
// Contradiction if: high similarity + asymmetric negation
|
||||
negation_diff >= 1 && similarity >= 0.7
|
||||
}
|
||||
|
||||
/// Score a memory against its related memories.
|
||||
///
|
||||
/// This is the core scoring function that:
|
||||
/// 1. Classifies related memories as confirmations or contradictions
|
||||
/// 2. Builds PLN truth values for each
|
||||
/// 3. Applies evidence-based scoring
|
||||
/// 4. Computes ECAN attention values
|
||||
/// 5. Categorizes the result
|
||||
pub fn score_memory(
|
||||
config: &ScorerConfig,
|
||||
candidate_content: &str,
|
||||
related: &[RelatedMemory],
|
||||
existing_sti: Option<f32>,
|
||||
existing_lti: Option<f32>,
|
||||
) -> ScoringResult {
|
||||
// Build base truth value from config
|
||||
let base_tv = TruthValue::new(config.pln_base_confidence, config.pln_base_confidence);
|
||||
|
||||
// Classify related memories
|
||||
let mut confirmations: Vec<TruthValue> = Vec::new();
|
||||
let mut contradictions: Vec<TruthValue> = Vec::new();
|
||||
|
||||
for related_mem in related {
|
||||
let strength = related_mem.truth_value.unwrap_or(0.5);
|
||||
let confidence = related_mem.truth_confidence.unwrap_or(0.3);
|
||||
|
||||
// Scale confidence by similarity — closer matches are stronger evidence
|
||||
let scaled_confidence = confidence * related_mem.similarity;
|
||||
|
||||
let tv = TruthValue::new(strength, scaled_confidence);
|
||||
|
||||
if is_contradiction(candidate_content, &related_mem.content, related_mem.similarity) {
|
||||
contradictions.push(tv);
|
||||
} else if related_mem.similarity >= config.contradiction_threshold {
|
||||
// High similarity + not contradiction = confirmation
|
||||
confirmations.push(tv);
|
||||
}
|
||||
// Memories below contradiction_threshold similarity are ignored
|
||||
// (not similar enough to be meaningful evidence)
|
||||
}
|
||||
|
||||
// Apply PLN evidence-based scoring
|
||||
let scored_tv = pln::score_with_evidence(base_tv, &confirmations, &contradictions);
|
||||
|
||||
// Compute ECAN values
|
||||
let (ecan_sti, ecan_lti) = if let (Some(sti), Some(lti)) = (existing_sti, existing_lti) {
|
||||
// Re-scoring: run ECAN cycle on existing values
|
||||
config.ecan.cycle(sti, lti, scored_tv.strength)
|
||||
} else {
|
||||
// First scoring: initialize from truth value
|
||||
config.ecan.initialize(scored_tv.strength, scored_tv.confidence)
|
||||
};
|
||||
|
||||
// Categorize
|
||||
let category = categorize(
|
||||
scored_tv.strength,
|
||||
&confirmations,
|
||||
&contradictions,
|
||||
config.verification_threshold,
|
||||
);
|
||||
|
||||
ScoringResult {
|
||||
truth_value: scored_tv.strength,
|
||||
truth_confidence: scored_tv.confidence,
|
||||
category,
|
||||
ecan_sti,
|
||||
ecan_lti,
|
||||
confirmation_count: confirmations.len(),
|
||||
contradiction_count: contradictions.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Categorize a scored memory based on its truth value and evidence.
|
||||
fn categorize(
|
||||
truth_value: f32,
|
||||
confirmations: &[TruthValue],
|
||||
contradictions: &[TruthValue],
|
||||
verification_threshold: f32,
|
||||
) -> TruthCategory {
|
||||
if !contradictions.is_empty() {
|
||||
return TruthCategory::Contradicted;
|
||||
}
|
||||
if truth_value >= verification_threshold && !confirmations.is_empty() {
|
||||
return TruthCategory::Verified;
|
||||
}
|
||||
if truth_value >= 0.5 {
|
||||
return TruthCategory::Plausible;
|
||||
}
|
||||
TruthCategory::Unverified
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn default_config() -> ScorerConfig {
|
||||
ScorerConfig::default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_no_related_memories() {
|
||||
let config = default_config();
|
||||
let result = score_memory(&config, "test claim", &[], None, None);
|
||||
|
||||
// With no evidence, should get base confidence as plausible
|
||||
assert!(result.truth_value > 0.0);
|
||||
assert!(result.truth_confidence > 0.0);
|
||||
assert_eq!(result.category, TruthCategory::Plausible);
|
||||
assert_eq!(result.confirmation_count, 0);
|
||||
assert_eq!(result.contradiction_count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_with_confirmations_boosts_truth() {
|
||||
let config = default_config();
|
||||
let related = vec![
|
||||
RelatedMemory {
|
||||
similarity: 0.92,
|
||||
content: "supporting evidence".to_string(),
|
||||
truth_value: Some(0.9),
|
||||
truth_confidence: Some(0.8),
|
||||
},
|
||||
RelatedMemory {
|
||||
similarity: 0.88,
|
||||
content: "more support".to_string(),
|
||||
truth_value: Some(0.85),
|
||||
truth_confidence: Some(0.7),
|
||||
},
|
||||
];
|
||||
|
||||
let result = score_memory(&config, "verified claim", &related, None, None);
|
||||
|
||||
assert!(result.confirmation_count >= 1);
|
||||
assert_eq!(result.contradiction_count, 0);
|
||||
assert_eq!(result.category, TruthCategory::Verified);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_with_contradiction_detected() {
|
||||
let config = default_config();
|
||||
let related = vec![RelatedMemory {
|
||||
similarity: 0.85,
|
||||
content: "This is not true and incorrect".to_string(),
|
||||
truth_value: Some(0.8),
|
||||
truth_confidence: Some(0.7),
|
||||
}];
|
||||
|
||||
let result = score_memory(
|
||||
&config,
|
||||
"This is true and correct",
|
||||
&related,
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
assert!(result.contradiction_count >= 1);
|
||||
assert_eq!(result.category, TruthCategory::Contradicted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_with_existing_ecan_runs_cycle() {
|
||||
let config = default_config();
|
||||
let result = score_memory(
|
||||
&config,
|
||||
"existing memory",
|
||||
&[],
|
||||
Some(0.8),
|
||||
Some(0.5),
|
||||
);
|
||||
|
||||
// ECAN cycle should decay STI
|
||||
assert!(result.ecan_sti < 0.8, "STI should decay: {}", result.ecan_sti);
|
||||
assert!(result.ecan_sti > 0.0);
|
||||
assert!(result.ecan_lti > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn score_first_time_initializes_ecan() {
|
||||
let config = default_config();
|
||||
let result = score_memory(&config, "new memory", &[], None, None);
|
||||
|
||||
// ECAN should be initialized from truth value
|
||||
assert!(result.ecan_sti > 0.0, "STI should be initialized");
|
||||
assert!(result.ecan_lti >= 0.0, "LTI should be initialized");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categorize_verified() {
|
||||
let cat = categorize(
|
||||
0.9,
|
||||
&[TruthValue::new(0.8, 0.7)],
|
||||
&[],
|
||||
0.8,
|
||||
);
|
||||
assert_eq!(cat, TruthCategory::Verified);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categorize_plausible() {
|
||||
let cat = categorize(0.6, &[], &[], 0.8);
|
||||
assert_eq!(cat, TruthCategory::Plausible);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categorize_unverified() {
|
||||
let cat = categorize(0.3, &[], &[], 0.8);
|
||||
assert_eq!(cat, TruthCategory::Unverified);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categorize_contradicted_takes_priority() {
|
||||
// Even with high truth value, contradictions = contradicted
|
||||
let cat = categorize(
|
||||
0.95,
|
||||
&[TruthValue::new(0.9, 0.8)],
|
||||
&[TruthValue::new(0.7, 0.6)],
|
||||
0.8,
|
||||
);
|
||||
assert_eq!(cat, TruthCategory::Contradicted);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_contradiction_negation_asymmetry() {
|
||||
assert!(is_contradiction(
|
||||
"The sky is blue",
|
||||
"The sky is not blue",
|
||||
0.85,
|
||||
));
|
||||
assert!(!is_contradiction(
|
||||
"The sky is blue",
|
||||
"The ocean is blue",
|
||||
0.80,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_contradiction_low_similarity_returns_false() {
|
||||
assert!(!is_contradiction(
|
||||
"Apples are healthy",
|
||||
"Cars are not fast",
|
||||
0.3,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoring_result_bounded() {
|
||||
let config = default_config();
|
||||
let result = score_memory(&config, "test", &[], None, None);
|
||||
|
||||
assert!(result.truth_value >= 0.0 && result.truth_value <= 1.0);
|
||||
assert!(result.truth_confidence >= 0.0 && result.truth_confidence <= 1.0);
|
||||
assert!(result.ecan_sti >= 0.0 && result.ecan_sti <= 1.0);
|
||||
assert!(result.ecan_lti >= 0.0 && result.ecan_lti <= 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truth_category_display() {
|
||||
assert_eq!(TruthCategory::Verified.as_str(), "verified");
|
||||
assert_eq!(TruthCategory::Plausible.as_str(), "plausible");
|
||||
assert_eq!(TruthCategory::Unverified.as_str(), "unverified");
|
||||
assert_eq!(TruthCategory::Contradicted.as_str(), "contradicted");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user