"""
Advanced Evaluation Example

This script demonstrates the core evaluation patterns from the advanced-evaluation skill.
It uses pseudocode that works across Python environments without specific dependencies.
"""

# =============================================================================
# DIRECT SCORING EXAMPLE
# =============================================================================

def direct_scoring_example():
    """
    Direct scoring: Rate a single response against defined criteria.
    Best for objective criteria like accuracy, completeness, instruction following.
    """
    
    # Input
    prompt = "Explain quantum entanglement to a high school student"
    response = """
    Quantum entanglement is like having two magical coins that are connected. 
    When you flip one and it lands on heads, the other instantly shows tails, 
    no matter how far apart they are. Scientists call this "spooky action at a distance."
    """
    
    criteria = [
        {"name": "Accuracy", "description": "Scientific correctness", "weight": 0.4},
        {"name": "Clarity", "description": "Understandable for audience", "weight": 0.3},
        {"name": "Engagement", "description": "Interesting and memorable", "weight": 0.3}
    ]
    
    # System prompt for the evaluator
    system_prompt = """You are an expert evaluator. Assess the response against each criterion.

For each criterion:
1. Find specific evidence in the response
2. Score according to the rubric (1-5 scale)
3. Justify your score with evidence
4. Suggest one specific improvement

Be objective and consistent. Base scores on explicit evidence."""
    
    # User prompt structure
    user_prompt = f"""## Original Prompt
{prompt}

## Response to Evaluate
{response}

## Criteria
1. **Accuracy** (weight: 0.4): Scientific correctness
2. **Clarity** (weight: 0.3): Understandable for audience  
3. **Engagement** (weight: 0.3): Interesting and memorable

## Output Format
Respond with valid JSON:
{{
  "scores": [
    {{
      "criterion": "Accuracy",
      "score": 4,
      "evidence": ["quote or observation"],
      "justification": "why this score",
      "improvement": "specific suggestion"
    }}
  ],
  "summary": {{
    "assessment": "overall quality summary",
    "strengths": ["strength 1"],
    "weaknesses": ["weakness 1"]
  }}
}}"""
    
    # Expected output structure
    expected_output = {
        "scores": [
            {
                "criterion": "Accuracy",
                "score": 4,
                "evidence": ["Correctly uses analogy", "Mentions spooky action at a distance"],
                "justification": "Core concept is correct, analogy is appropriate",
                "improvement": "Could mention it's a quantum mechanical phenomenon"
            },
            {
                "criterion": "Clarity", 
                "score": 5,
                "evidence": ["Simple coin analogy", "No jargon"],
                "justification": "Appropriate for high school level",
                "improvement": "None needed"
            },
            {
                "criterion": "Engagement",
                "score": 4,
                "evidence": ["Magical coins", "Spooky action quote"],
                "justification": "Memorable imagery and Einstein quote",
                "improvement": "Could add a real-world application"
            }
        ],
        "summary": {
            "assessment": "Good explanation suitable for the target audience",
            "strengths": ["Clear analogy", "Age-appropriate language"],
            "weaknesses": ["Could be more comprehensive"]
        }
    }
    
    # Calculate weighted score
    total_weight = sum(c["weight"] for c in criteria)
    weighted_score = sum(
        s["score"] * next(c["weight"] for c in criteria if c["name"] == s["criterion"])
        for s in expected_output["scores"]
    ) / total_weight
    
    print(f"Weighted Score: {weighted_score:.2f}/5")
    return expected_output


# =============================================================================
# PAIRWISE COMPARISON WITH POSITION BIAS MITIGATION
# =============================================================================

def pairwise_comparison_example():
    """
    Pairwise comparison: Compare two responses and select the better one.
    Includes position swapping to mitigate position bias.
    Best for subjective preferences like tone, style, persuasiveness.
    """
    
    prompt = "Explain machine learning to a beginner"
    
    response_a = """
    Machine learning is a subset of artificial intelligence that enables 
    systems to learn and improve from experience without being explicitly 
    programmed. It uses statistical techniques to give computers the ability 
    to identify patterns in data.
    """
    
    response_b = """
    Imagine teaching a dog a new trick. You show the dog what to do, give 
    treats when it's right, and eventually it learns. Machine learning works 
    similarly - we show computers lots of examples, tell them when they're 
    right, and they learn to recognize patterns on their own.
    """
    
    criteria = ["clarity", "accessibility", "accuracy"]
    
    # System prompt emphasizing bias awareness
    system_prompt = """You are an expert evaluator comparing two AI responses.

CRITICAL INSTRUCTIONS:
- Do NOT prefer responses because they are longer
- Do NOT prefer responses based on position (first vs second)
- Focus ONLY on quality according to the specified criteria
- Ties are acceptable when responses are genuinely equivalent"""
    
    # First pass: A first, B second
    def evaluate_pass(first_response, second_response, first_label, second_label):
        user_prompt = f"""## Original Prompt
{prompt}

## Response {first_label}
{first_response}

## Response {second_label}
{second_response}

## Comparison Criteria
{', '.join(criteria)}

## Output Format
{{
  "comparison": [
    {{"criterion": "clarity", "winner": "A|B|TIE", "reasoning": "..."}}
  ],
  "result": {{
    "winner": "A|B|TIE",
    "confidence": 0.0-1.0,
    "reasoning": "overall reasoning"
  }}
}}"""
        return user_prompt
    
    # Position bias mitigation protocol
    print("Pass 1: A in first position")
    pass1_result = {"winner": "B", "confidence": 0.8}
    
    print("Pass 2: B in first position (swapped)")
    pass2_result = {"winner": "A", "confidence": 0.75}  # A because B was first
    
    # Map pass2 result back (swap labels)
    def map_winner(winner):
        return {"A": "B", "B": "A", "TIE": "TIE"}[winner]
    
    pass2_mapped = map_winner(pass2_result["winner"])
    print(f"Pass 2 mapped winner: {pass2_mapped}")
    
    # Check consistency
    consistent = pass1_result["winner"] == pass2_mapped
    
    if consistent:
        final_result = {
            "winner": pass1_result["winner"],
            "confidence": (pass1_result["confidence"] + pass2_result["confidence"]) / 2,
            "position_consistent": True
        }
    else:
        final_result = {
            "winner": "TIE",
            "confidence": 0.5,
            "position_consistent": False,
            "bias_detected": True
        }
    
    print(f"\nFinal Result: {final_result}")
    return final_result


# =============================================================================
# RUBRIC GENERATION
# =============================================================================

def rubric_generation_example():
    """
    Generate a domain-specific scoring rubric.
    Rubrics reduce evaluation variance by 40-60%.
    """
    
    criterion_name = "Code Readability"
    criterion_description = "How easy the code is to understand and maintain"
    domain = "software engineering"
    scale = "1-5"
    strictness = "balanced"
    
    system_prompt = f"""You are an expert in creating evaluation rubrics.
Create clear, actionable rubrics with distinct boundaries between levels.

Strictness: {strictness}
- lenient: Lower bar for passing scores
- balanced: Fair, typical expectations
- strict: High standards, critical evaluation"""
    
    user_prompt = f"""Create a scoring rubric for:

**Criterion**: {criterion_name}
**Description**: {criterion_description}
**Scale**: {scale}
**Domain**: {domain}

Generate:
1. Clear descriptions for each score level
2. Specific characteristics that define each level
3. Brief example text for each level
4. General scoring guidelines
5. Edge cases with guidance"""
    
    # Expected rubric structure
    rubric = {
        "criterion": criterion_name,
        "scale": {"min": 1, "max": 5},
        "levels": [
            {
                "score": 1,
                "label": "Poor",
                "description": "Code is difficult to understand without significant effort",
                "characteristics": [
                    "No meaningful variable or function names",
                    "No comments or documentation", 
                    "Deeply nested or convoluted logic"
                ],
                "example": "def f(x): return x[0]*x[1]+x[2]"
            },
            {
                "score": 3,
                "label": "Adequate", 
                "description": "Code is understandable with some effort",
                "characteristics": [
                    "Most variables have meaningful names",
                    "Basic comments for complex sections",
                    "Logic is followable but could be cleaner"
                ],
                "example": "def calc_total(items): # calculate sum\n    total = 0\n    for i in items: total += i\n    return total"
            },
            {
                "score": 5,
                "label": "Excellent",
                "description": "Code is immediately clear and maintainable",
                "characteristics": [
                    "All names are descriptive and consistent",
                    "Comprehensive documentation",
                    "Clean, modular structure"
                ],
                "example": "def calculate_total_price(items: List[Item]) -> Decimal:\n    '''Calculate the total price of all items.'''\n    return sum(item.price for item in items)"
            }
        ],
        "scoring_guidelines": [
            "Focus on readability, not cleverness",
            "Consider the intended audience (team skill level)",
            "Consistency matters more than style preference"
        ],
        "edge_cases": [
            {
                "situation": "Code uses domain-specific abbreviations",
                "guidance": "Score based on readability for domain experts, not general audience"
            },
            {
                "situation": "Code is auto-generated",
                "guidance": "Apply same standards but note in evaluation"
            }
        ]
    }
    
    print("Generated Rubric:")
    for level in rubric["levels"]:
        print(f"  {level['score']}: {level['label']} - {level['description']}")
    
    return rubric


# =============================================================================
# MAIN
# =============================================================================

if __name__ == "__main__":
    print("=" * 60)
    print("DIRECT SCORING EXAMPLE")
    print("=" * 60)
    direct_scoring_example()
    
    print("\n" + "=" * 60)
    print("PAIRWISE COMPARISON EXAMPLE")
    print("=" * 60)
    pairwise_comparison_example()
    
    print("\n" + "=" * 60)
    print("RUBRIC GENERATION EXAMPLE")
    print("=" * 60)
    rubric_generation_example()

