mirror of
https://gitea.ingwaz.work/Ingwaz/openbrain-mcp.git
synced 2026-06-15 22:07:08 +00:00
feat(truth): add scoring pipeline module (#34)
Implement truth scoring orchestrator that ties PLN + ECAN together: New types: - TruthCategory: verified/plausible/unverified/contradicted enum - ScorerConfig: pipeline configuration (thresholds, ECAN params) - RelatedMemory: cross-reference result with similarity + existing scores - ScoringResult: complete scoring output with TV, confidence, category, ECAN Core functions: - score_memory(): orchestrates evidence classification, PLN scoring, ECAN computation, and categorization - is_contradiction(): heuristic negation-asymmetry detection - categorize(): rule-based category assignment Scoring pipeline: 1. Classify related memories as confirmations or contradictions 2. Scale confidence by cosine similarity (closer = stronger evidence) 3. Apply PLN score_with_evidence for truth value computation 4. Run ECAN cycle (re-score) or initialize (first score) 5. Categorize based on TV, confirmations, and contradictions 13 unit tests covering all scoring paths, categories, contradiction detection, ECAN initialization/cycling, and output bounds. Part of #29
This commit is contained in:
@@ -13,3 +13,4 @@
|
|||||||
|
|
||||||
pub mod ecan;
|
pub mod ecan;
|
||||||
pub mod pln;
|
pub mod pln;
|
||||||
|
pub mod scorer;
|
||||||
|
|||||||
396
src/truth/scorer.rs
Normal file
396
src/truth/scorer.rs
Normal file
@@ -0,0 +1,396 @@
|
|||||||
|
//! Truth scoring pipeline — orchestrates PLN, ECAN, and cross-referencing
|
||||||
|
//! to produce truth scores for memories.
|
||||||
|
//!
|
||||||
|
//! This is the core intelligence module that evaluates each memory by:
|
||||||
|
//! 1. Finding related memories via vector similarity (cross-referencing)
|
||||||
|
//! 2. Classifying related memories as confirmations or contradictions
|
||||||
|
//! 3. Applying PLN evidence-based scoring
|
||||||
|
//! 4. Computing ECAN attention values (STI/LTI)
|
||||||
|
//! 5. Categorizing the result
|
||||||
|
|
||||||
|
use crate::truth::ecan::EcanParams;
|
||||||
|
use crate::truth::pln::{self, TruthValue};
|
||||||
|
|
||||||
|
/// Truth category assigned after scoring.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum TruthCategory {
|
||||||
|
/// TV ≥ verification_threshold AND has confirmations
|
||||||
|
Verified,
|
||||||
|
/// TV ≥ 0.5
|
||||||
|
Plausible,
|
||||||
|
/// TV < 0.5 AND no contradictions
|
||||||
|
Unverified,
|
||||||
|
/// Contradictions detected
|
||||||
|
Contradicted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl TruthCategory {
|
||||||
|
pub fn as_str(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
TruthCategory::Verified => "verified",
|
||||||
|
TruthCategory::Plausible => "plausible",
|
||||||
|
TruthCategory::Unverified => "unverified",
|
||||||
|
TruthCategory::Contradicted => "contradicted",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for TruthCategory {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str(self.as_str())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Configuration for the scoring pipeline.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ScorerConfig {
|
||||||
|
/// Base confidence for PLN scoring when no prior exists.
|
||||||
|
pub pln_base_confidence: f32,
|
||||||
|
/// Similarity threshold above which two memories are considered related.
|
||||||
|
pub contradiction_threshold: f32,
|
||||||
|
/// Truth value threshold for "verified" category.
|
||||||
|
pub verification_threshold: f32,
|
||||||
|
/// ECAN parameters.
|
||||||
|
pub ecan: EcanParams,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for ScorerConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
pln_base_confidence: 0.85,
|
||||||
|
contradiction_threshold: 0.85,
|
||||||
|
verification_threshold: 0.8,
|
||||||
|
ecan: EcanParams::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A related memory found during cross-referencing.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct RelatedMemory {
|
||||||
|
/// Cosine similarity to the candidate memory.
|
||||||
|
pub similarity: f32,
|
||||||
|
/// Content of the related memory.
|
||||||
|
pub content: String,
|
||||||
|
/// Existing truth value of the related memory (if scored).
|
||||||
|
pub truth_value: Option<f32>,
|
||||||
|
/// Existing confidence of the related memory (if scored).
|
||||||
|
pub truth_confidence: Option<f32>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Result of scoring a single memory.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ScoringResult {
|
||||||
|
pub truth_value: f32,
|
||||||
|
pub truth_confidence: f32,
|
||||||
|
pub category: TruthCategory,
|
||||||
|
pub ecan_sti: f32,
|
||||||
|
pub ecan_lti: f32,
|
||||||
|
pub confirmation_count: usize,
|
||||||
|
pub contradiction_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Determine whether two pieces of content are contradictory.
|
||||||
|
///
|
||||||
|
/// This uses a heuristic approach: if two memories are semantically similar
|
||||||
|
/// (high cosine similarity) but contain negation patterns, they likely
|
||||||
|
/// contradict each other.
|
||||||
|
///
|
||||||
|
/// A more sophisticated implementation could use an NLI model, but this
|
||||||
|
/// provides a reasonable baseline.
|
||||||
|
fn is_contradiction(candidate_content: &str, related_content: &str, similarity: f32) -> bool {
|
||||||
|
// High similarity is required — unrelated content can't contradict
|
||||||
|
if similarity < 0.6 {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let negation_markers = [
|
||||||
|
"not ", "never ", "no ", "false", "incorrect", "wrong",
|
||||||
|
"isn't", "aren't", "wasn't", "weren't", "don't", "doesn't",
|
||||||
|
"didn't", "won't", "wouldn't", "shouldn't", "couldn't",
|
||||||
|
"cannot", "can't", "unlikely", "impossible", "untrue",
|
||||||
|
];
|
||||||
|
|
||||||
|
let candidate_lower = candidate_content.to_lowercase();
|
||||||
|
let related_lower = related_content.to_lowercase();
|
||||||
|
|
||||||
|
let candidate_negations = negation_markers
|
||||||
|
.iter()
|
||||||
|
.filter(|m| candidate_lower.contains(*m))
|
||||||
|
.count();
|
||||||
|
let related_negations = negation_markers
|
||||||
|
.iter()
|
||||||
|
.filter(|m| related_lower.contains(*m))
|
||||||
|
.count();
|
||||||
|
|
||||||
|
// If one has significantly more negation markers than the other,
|
||||||
|
// and they're semantically similar, it suggests contradiction.
|
||||||
|
let negation_diff = (candidate_negations as i32 - related_negations as i32).unsigned_abs();
|
||||||
|
|
||||||
|
// Contradiction if: high similarity + asymmetric negation
|
||||||
|
negation_diff >= 1 && similarity >= 0.7
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Score a memory against its related memories.
|
||||||
|
///
|
||||||
|
/// This is the core scoring function that:
|
||||||
|
/// 1. Classifies related memories as confirmations or contradictions
|
||||||
|
/// 2. Builds PLN truth values for each
|
||||||
|
/// 3. Applies evidence-based scoring
|
||||||
|
/// 4. Computes ECAN attention values
|
||||||
|
/// 5. Categorizes the result
|
||||||
|
pub fn score_memory(
|
||||||
|
config: &ScorerConfig,
|
||||||
|
candidate_content: &str,
|
||||||
|
related: &[RelatedMemory],
|
||||||
|
existing_sti: Option<f32>,
|
||||||
|
existing_lti: Option<f32>,
|
||||||
|
) -> ScoringResult {
|
||||||
|
// Build base truth value from config
|
||||||
|
let base_tv = TruthValue::new(config.pln_base_confidence, config.pln_base_confidence);
|
||||||
|
|
||||||
|
// Classify related memories
|
||||||
|
let mut confirmations: Vec<TruthValue> = Vec::new();
|
||||||
|
let mut contradictions: Vec<TruthValue> = Vec::new();
|
||||||
|
|
||||||
|
for related_mem in related {
|
||||||
|
let strength = related_mem.truth_value.unwrap_or(0.5);
|
||||||
|
let confidence = related_mem.truth_confidence.unwrap_or(0.3);
|
||||||
|
|
||||||
|
// Scale confidence by similarity — closer matches are stronger evidence
|
||||||
|
let scaled_confidence = confidence * related_mem.similarity;
|
||||||
|
|
||||||
|
let tv = TruthValue::new(strength, scaled_confidence);
|
||||||
|
|
||||||
|
if is_contradiction(candidate_content, &related_mem.content, related_mem.similarity) {
|
||||||
|
contradictions.push(tv);
|
||||||
|
} else if related_mem.similarity >= config.contradiction_threshold {
|
||||||
|
// High similarity + not contradiction = confirmation
|
||||||
|
confirmations.push(tv);
|
||||||
|
}
|
||||||
|
// Memories below contradiction_threshold similarity are ignored
|
||||||
|
// (not similar enough to be meaningful evidence)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply PLN evidence-based scoring
|
||||||
|
let scored_tv = pln::score_with_evidence(base_tv, &confirmations, &contradictions);
|
||||||
|
|
||||||
|
// Compute ECAN values
|
||||||
|
let (ecan_sti, ecan_lti) = if let (Some(sti), Some(lti)) = (existing_sti, existing_lti) {
|
||||||
|
// Re-scoring: run ECAN cycle on existing values
|
||||||
|
config.ecan.cycle(sti, lti, scored_tv.strength)
|
||||||
|
} else {
|
||||||
|
// First scoring: initialize from truth value
|
||||||
|
config.ecan.initialize(scored_tv.strength, scored_tv.confidence)
|
||||||
|
};
|
||||||
|
|
||||||
|
// Categorize
|
||||||
|
let category = categorize(
|
||||||
|
scored_tv.strength,
|
||||||
|
&confirmations,
|
||||||
|
&contradictions,
|
||||||
|
config.verification_threshold,
|
||||||
|
);
|
||||||
|
|
||||||
|
ScoringResult {
|
||||||
|
truth_value: scored_tv.strength,
|
||||||
|
truth_confidence: scored_tv.confidence,
|
||||||
|
category,
|
||||||
|
ecan_sti,
|
||||||
|
ecan_lti,
|
||||||
|
confirmation_count: confirmations.len(),
|
||||||
|
contradiction_count: contradictions.len(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Categorize a scored memory based on its truth value and evidence.
|
||||||
|
fn categorize(
|
||||||
|
truth_value: f32,
|
||||||
|
confirmations: &[TruthValue],
|
||||||
|
contradictions: &[TruthValue],
|
||||||
|
verification_threshold: f32,
|
||||||
|
) -> TruthCategory {
|
||||||
|
if !contradictions.is_empty() {
|
||||||
|
return TruthCategory::Contradicted;
|
||||||
|
}
|
||||||
|
if truth_value >= verification_threshold && !confirmations.is_empty() {
|
||||||
|
return TruthCategory::Verified;
|
||||||
|
}
|
||||||
|
if truth_value >= 0.5 {
|
||||||
|
return TruthCategory::Plausible;
|
||||||
|
}
|
||||||
|
TruthCategory::Unverified
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn default_config() -> ScorerConfig {
|
||||||
|
ScorerConfig::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_no_related_memories() {
|
||||||
|
let config = default_config();
|
||||||
|
let result = score_memory(&config, "test claim", &[], None, None);
|
||||||
|
|
||||||
|
// With no evidence, should get base confidence as plausible
|
||||||
|
assert!(result.truth_value > 0.0);
|
||||||
|
assert!(result.truth_confidence > 0.0);
|
||||||
|
assert_eq!(result.category, TruthCategory::Plausible);
|
||||||
|
assert_eq!(result.confirmation_count, 0);
|
||||||
|
assert_eq!(result.contradiction_count, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_with_confirmations_boosts_truth() {
|
||||||
|
let config = default_config();
|
||||||
|
let related = vec![
|
||||||
|
RelatedMemory {
|
||||||
|
similarity: 0.92,
|
||||||
|
content: "supporting evidence".to_string(),
|
||||||
|
truth_value: Some(0.9),
|
||||||
|
truth_confidence: Some(0.8),
|
||||||
|
},
|
||||||
|
RelatedMemory {
|
||||||
|
similarity: 0.88,
|
||||||
|
content: "more support".to_string(),
|
||||||
|
truth_value: Some(0.85),
|
||||||
|
truth_confidence: Some(0.7),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
let result = score_memory(&config, "verified claim", &related, None, None);
|
||||||
|
|
||||||
|
assert!(result.confirmation_count >= 1);
|
||||||
|
assert_eq!(result.contradiction_count, 0);
|
||||||
|
assert_eq!(result.category, TruthCategory::Verified);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_with_contradiction_detected() {
|
||||||
|
let config = default_config();
|
||||||
|
let related = vec![RelatedMemory {
|
||||||
|
similarity: 0.85,
|
||||||
|
content: "This is not true and incorrect".to_string(),
|
||||||
|
truth_value: Some(0.8),
|
||||||
|
truth_confidence: Some(0.7),
|
||||||
|
}];
|
||||||
|
|
||||||
|
let result = score_memory(
|
||||||
|
&config,
|
||||||
|
"This is true and correct",
|
||||||
|
&related,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(result.contradiction_count >= 1);
|
||||||
|
assert_eq!(result.category, TruthCategory::Contradicted);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_with_existing_ecan_runs_cycle() {
|
||||||
|
let config = default_config();
|
||||||
|
let result = score_memory(
|
||||||
|
&config,
|
||||||
|
"existing memory",
|
||||||
|
&[],
|
||||||
|
Some(0.8),
|
||||||
|
Some(0.5),
|
||||||
|
);
|
||||||
|
|
||||||
|
// ECAN cycle should decay STI
|
||||||
|
assert!(result.ecan_sti < 0.8, "STI should decay: {}", result.ecan_sti);
|
||||||
|
assert!(result.ecan_sti > 0.0);
|
||||||
|
assert!(result.ecan_lti > 0.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn score_first_time_initializes_ecan() {
|
||||||
|
let config = default_config();
|
||||||
|
let result = score_memory(&config, "new memory", &[], None, None);
|
||||||
|
|
||||||
|
// ECAN should be initialized from truth value
|
||||||
|
assert!(result.ecan_sti > 0.0, "STI should be initialized");
|
||||||
|
assert!(result.ecan_lti >= 0.0, "LTI should be initialized");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn categorize_verified() {
|
||||||
|
let cat = categorize(
|
||||||
|
0.9,
|
||||||
|
&[TruthValue::new(0.8, 0.7)],
|
||||||
|
&[],
|
||||||
|
0.8,
|
||||||
|
);
|
||||||
|
assert_eq!(cat, TruthCategory::Verified);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn categorize_plausible() {
|
||||||
|
let cat = categorize(0.6, &[], &[], 0.8);
|
||||||
|
assert_eq!(cat, TruthCategory::Plausible);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn categorize_unverified() {
|
||||||
|
let cat = categorize(0.3, &[], &[], 0.8);
|
||||||
|
assert_eq!(cat, TruthCategory::Unverified);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn categorize_contradicted_takes_priority() {
|
||||||
|
// Even with high truth value, contradictions = contradicted
|
||||||
|
let cat = categorize(
|
||||||
|
0.95,
|
||||||
|
&[TruthValue::new(0.9, 0.8)],
|
||||||
|
&[TruthValue::new(0.7, 0.6)],
|
||||||
|
0.8,
|
||||||
|
);
|
||||||
|
assert_eq!(cat, TruthCategory::Contradicted);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn is_contradiction_negation_asymmetry() {
|
||||||
|
assert!(is_contradiction(
|
||||||
|
"The sky is blue",
|
||||||
|
"The sky is not blue",
|
||||||
|
0.85,
|
||||||
|
));
|
||||||
|
assert!(!is_contradiction(
|
||||||
|
"The sky is blue",
|
||||||
|
"The ocean is blue",
|
||||||
|
0.80,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn is_contradiction_low_similarity_returns_false() {
|
||||||
|
assert!(!is_contradiction(
|
||||||
|
"Apples are healthy",
|
||||||
|
"Cars are not fast",
|
||||||
|
0.3,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn scoring_result_bounded() {
|
||||||
|
let config = default_config();
|
||||||
|
let result = score_memory(&config, "test", &[], None, None);
|
||||||
|
|
||||||
|
assert!(result.truth_value >= 0.0 && result.truth_value <= 1.0);
|
||||||
|
assert!(result.truth_confidence >= 0.0 && result.truth_confidence <= 1.0);
|
||||||
|
assert!(result.ecan_sti >= 0.0 && result.ecan_sti <= 1.0);
|
||||||
|
assert!(result.ecan_lti >= 0.0 && result.ecan_lti <= 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn truth_category_display() {
|
||||||
|
assert_eq!(TruthCategory::Verified.as_str(), "verified");
|
||||||
|
assert_eq!(TruthCategory::Plausible.as_str(), "plausible");
|
||||||
|
assert_eq!(TruthCategory::Unverified.as_str(), "unverified");
|
||||||
|
assert_eq!(TruthCategory::Contradicted.as_str(), "contradicted");
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user