Ranged scoring with LLM judges

from adaptive_harmony import StringThread
from adaptive_harmony.graders.range_judge import RangeJudgeShot, SubrangeExpectations

# Define the scoring criteria
criteria = """
Score the quality of the completion on a scale of 1-5:

1 - Irrelevant or off-topic: Completion does not address the prompt or is contradictory.
2 - Incomplete: Completion touches on the topic but lacks detail or clarity.
3 - Adequate: Completion covers the main points but is missing some depth or supporting details.
4 - Good: Completion is relevant, clear, and covers most key aspects with supporting details.
5 - Excellent: Completion is thorough, well-structured, and provides comprehensive information with clear reasoning.
"""

# Define the evaluation steps
evaluation_steps = [
    "Does the completion directly address the prompt?",
    "Is the information relevant and accurate?",
    "How complete is the answer? Are key points covered?",
    "Is the information well-organized and easy to follow?",
    "Are claims supported with reasoning or examples?",
    "Assign a score from 1-5 based on the rubric above.",
]

print("📋 Criteria:")
print(criteria)
print("📝 Evaluation steps:")
for i, step in enumerate(evaluation_steps, 1):
    print(f"  {i}. {step}")

# Define subrange expectations: what each score range means
subrange_expectations = [
    SubrangeExpectations(
        subrange=(1, 2),
        expectation="The response is off-topic, incomplete, or of very low quality"
    ),
    SubrangeExpectations(
        subrange=(3, 3),
        expectation="The response covers the main points adequately but lacks depth or polish"
    ),
    SubrangeExpectations(
        subrange=(4, 5),
        expectation="The response is comprehensive, well-structured, and demonstrates expert understanding"
    ),
]

print("🎯 Subrange expectations:")
for se in subrange_expectations:
    print(f"  📊 Scores {se.subrange[0]}-{se.subrange[1]}: {se.expectation}")

# Few-shot examples for the judge
prompt = "Explain the difference between correlation and causation."

# Example 1: Score 1 (Irrelevant)
score_1_completion = "Correlation is when two things happen together. Causation is when something causes something else."
score_1_thread = StringThread([
    ("user", prompt),
    ("assistant", score_1_completion)
])

# Example 2: Score 3 (Adequate)
score_3_completion = (
    "Correlation means two variables move together (both go up or both go down). "
    "Causation means one variable actually causes the change in the other. "
    "A classic example is ice cream sales and drowning deaths—both increase in summer, but ice cream doesn't cause drowning."
)
score_3_thread = StringThread([
    ("user", prompt),
    ("assistant", score_3_completion)
])

# Example 3: Score 5 (Excellent)
score_5_completion = (
    "Correlation describes a statistical relationship between two variables where they tend to move together. "
    "When one increases, the other tends to increase (positive correlation) or decrease (negative correlation). "
    "However, correlation does not imply causation. Two variables can be correlated for several reasons:\n"
    "1. One causes the other (genuine causal link)\n"
    "2. A third variable causes both (confounding variable)\n"
    "3. The relationship is coincidental\n\n"
    "Example: Ice cream sales and drowning deaths are positively correlated, both peak in summer. "
    "But ice cream doesn't cause drowning—warm weather is the confounding variable that causes both. "
    "To establish causation, you need controlled experiments or causal inference methods that rule out confounds."
)
score_5_thread = StringThread([
    ("user", prompt),
    ("assistant", score_5_completion)
])

# Create RangeJudgeShot objects
shots = [
    RangeJudgeShot(
        thread=score_1_thread,
        reasoning="This explanation is too brief and lacks the key insight that correlation does not imply causation. It provides only basic definitions without examples or nuance.",
        score=1
    ),
    RangeJudgeShot(
        thread=score_3_thread,
        reasoning="This explanation covers the main distinction and provides a useful example. However, it could go deeper by explaining confounding variables more clearly.",
        score=3
    ),
    RangeJudgeShot(
        thread=score_5_thread,
        reasoning="This is a comprehensive explanation that defines both concepts clearly, provides multiple reasons for correlation without causation, includes a concrete example, and explains the role of confounding variables. The structure is logical and easy to follow.",
        score=5
    ),
]

print(f"🎓 Created {len(shots)} few-shot examples:")
for shot in shots:
    print(f"  📌 Score {shot.score}: {len(shot.thread.last_content())} characters")

from adaptive_harmony.graders import RangeJudgeGrader

# Requires an InferenceModel — shown as reference only
grader = RangeJudgeGrader(
    grader_key="completion-quality",
    model=inference_model,  # InferenceModel instance
    criteria=criteria,
    score_range=(1, 5),
    evaluation_steps=evaluation_steps,
    subrange_expectations=subrange_expectations,
    shots=shots,
    normalize_score=True,  # Scores 1-5 → 0.0-1.0
)

score_range = (1, 5)

# Validate subrange expectations cover the full score range
all_covered = set()
for se in subrange_expectations:
    for s in range(se.subrange[0], se.subrange[1] + 1):
        all_covered.add(s)
assert all_covered == set(range(score_range[0], score_range[1] + 1)), \
    f"Subranges don't cover full range: missing {set(range(*score_range)) - all_covered}"
print(f"✅ Subrange expectations cover the full {score_range[0]}-{score_range[1]} range")

# Validate few-shot examples have scores within range
for shot in shots:
    assert score_range[0] <= shot.score <= score_range[1], \
        f"Shot score {shot.score} outside range {score_range}"
    assert len(shot.reasoning) > 0, "Shot must have reasoning"
print(f"✅ All {len(shots)} few-shot scores are within [{score_range[0]}, {score_range[1]}]")

# Demonstrate normalization: raw score → [0, 1]
def normalize(raw_score, lo, hi):
    return (raw_score - lo) / (hi - lo)

print(f"\n📊 Score normalization (range {score_range[0]}-{score_range[1]}):")
for raw in range(score_range[0], score_range[1] + 1):
    norm = normalize(raw, *score_range)
    print(f"  Raw {raw} → Normalized {norm:.2f}")