first commit

app/services/cat_selection.py

@@ -0,0 +1,702 @@
+"""
+CAT (Computerized Adaptive Testing) Selection Service.
+
+Implements adaptive item selection algorithms for IRT-based testing.
+Supports three modes: CTT (fixed), IRT (adaptive), and hybrid.
+"""
+
+import math
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Literal, Optional
+
+from sqlalchemy import and_, not_, or_, select, func
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.models import Item, Session, Tryout, UserAnswer
+from app.services.irt_calibration import (
+    calculate_item_information,
+    estimate_b_from_ctt_p,
+    estimate_theta_mle,
+    update_theta_after_response,
+)
+
+
+class CATSelectionError(Exception):
+    """Exception raised for CAT selection errors."""
+    pass
+
+
+@dataclass
+class NextItemResult:
+    """Result of next item selection."""
+    item: Optional[Item]
+    selection_method: str  # 'fixed', 'adaptive', 'hybrid'
+    slot: Optional[int]
+    level: Optional[str]
+    reason: str  # Why this item was selected
+
+
+@dataclass
+class TerminationCheck:
+    """Result of termination condition check."""
+    should_terminate: bool
+    reason: str
+    items_answered: int
+    current_se: Optional[float]
+    max_items: Optional[int]
+    se_threshold_met: bool
+
+
+# Default SE threshold for termination
+DEFAULT_SE_THRESHOLD = 0.5
+# Default max items if not configured
+DEFAULT_MAX_ITEMS = 50
+
+
+async def get_next_item_fixed(
+    db: AsyncSession,
+    session_id: str,
+    tryout_id: str,
+    website_id: int,
+    level_filter: Optional[str] = None
+) -> NextItemResult:
+    """
+    Get next item in fixed order (CTT mode).
+    
+    Returns items in slot order (1, 2, 3, ...).
+    Filters by level if specified.
+    Checks if student already answered this item.
+    
+    Args:
+        db: Database session
+        session_id: Session identifier
+        tryout_id: Tryout identifier
+        website_id: Website identifier
+        level_filter: Optional difficulty level filter ('mudah', 'sedang', 'sulit')
+        
+    Returns:
+        NextItemResult with selected item or None if no more items
+    """
+    # Get session to find current position and answered items
+    session_query = select(Session).where(Session.session_id == session_id)
+    session_result = await db.execute(session_query)
+    session = session_result.scalar_one_or_none()
+    
+    if not session:
+        raise CATSelectionError(f"Session {session_id} not found")
+    
+    # Get all item IDs already answered by this user in this session
+    answered_query = select(UserAnswer.item_id).where(
+        UserAnswer.session_id == session_id
+    )
+    answered_result = await db.execute(answered_query)
+    answered_item_ids = [row[0] for row in answered_result.all()]
+    
+    # Build query for available items
+    query = (
+        select(Item)
+        .where(
+            Item.tryout_id == tryout_id,
+            Item.website_id == website_id
+        )
+        .order_by(Item.slot, Item.level)
+    )
+    
+    # Apply level filter if specified
+    if level_filter:
+        query = query.where(Item.level == level_filter)
+    
+    # Exclude already answered items
+    if answered_item_ids:
+        query = query.where(not_(Item.id.in_(answered_item_ids)))
+    
+    result = await db.execute(query)
+    items = result.scalars().all()
+    
+    if not items:
+        return NextItemResult(
+            item=None,
+            selection_method="fixed",
+            slot=None,
+            level=None,
+            reason="No more items available"
+        )
+    
+    # Return first available item (lowest slot)
+    next_item = items[0]
+    
+    return NextItemResult(
+        item=next_item,
+        selection_method="fixed",
+        slot=next_item.slot,
+        level=next_item.level,
+        reason=f"Fixed order selection - slot {next_item.slot}"
+    )
+
+
+async def get_next_item_adaptive(
+    db: AsyncSession,
+    session_id: str,
+    tryout_id: str,
+    website_id: int,
+    ai_generation_enabled: bool = False,
+    level_filter: Optional[str] = None
+) -> NextItemResult:
+    """
+    Get next item using adaptive selection (IRT mode).
+    
+    Finds item where b ≈ current theta.
+    Only uses calibrated items (calibrated=True).
+    Filters: student hasn't answered this item.
+    Filters: AI-generated items only if AI generation is enabled.
+    
+    Args:
+        db: Database session
+        session_id: Session identifier
+        tryout_id: Tryout identifier
+        website_id: Website identifier
+        ai_generation_enabled: Whether to include AI-generated items
+        level_filter: Optional difficulty level filter
+        
+    Returns:
+        NextItemResult with selected item or None if no suitable items
+    """
+    # Get session for current theta
+    session_query = select(Session).where(Session.session_id == session_id)
+    session_result = await db.execute(session_query)
+    session = session_result.scalar_one_or_none()
+    
+    if not session:
+        raise CATSelectionError(f"Session {session_id} not found")
+    
+    # Get current theta (default to 0.0 for first item)
+    current_theta = session.theta if session.theta is not None else 0.0
+    
+    # Get all item IDs already answered by this user in this session
+    answered_query = select(UserAnswer.item_id).where(
+        UserAnswer.session_id == session_id
+    )
+    answered_result = await db.execute(answered_query)
+    answered_item_ids = [row[0] for row in answered_result.all()]
+    
+    # Build query for available calibrated items
+    query = (
+        select(Item)
+        .where(
+            Item.tryout_id == tryout_id,
+            Item.website_id == website_id,
+            Item.calibrated == True  # Only calibrated items for IRT
+        )
+    )
+    
+    # Apply level filter if specified
+    if level_filter:
+        query = query.where(Item.level == level_filter)
+    
+    # Exclude already answered items
+    if answered_item_ids:
+        query = query.where(not_(Item.id.in_(answered_item_ids)))
+    
+    # Filter AI-generated items if AI generation is disabled
+    if not ai_generation_enabled:
+        query = query.where(Item.generated_by == 'manual')
+    
+    result = await db.execute(query)
+    items = result.scalars().all()
+    
+    if not items:
+        return NextItemResult(
+            item=None,
+            selection_method="adaptive",
+            slot=None,
+            level=None,
+            reason="No calibrated items available"
+        )
+    
+    # Find item with b closest to current theta
+    # Also consider item information (prefer items with higher information at current theta)
+    best_item = None
+    best_score = float('inf')
+    
+    for item in items:
+        if item.irt_b is None:
+            # Skip items without b parameter (shouldn't happen with calibrated=True)
+            continue
+        
+        # Calculate distance from theta
+        b_distance = abs(item.irt_b - current_theta)
+        
+        # Calculate item information at current theta
+        information = calculate_item_information(current_theta, item.irt_b)
+        
+        # Score: minimize distance, maximize information
+        # Use weighted combination: lower score is better
+        # Add small penalty for lower information
+        score = b_distance - (0.1 * information)
+        
+        if score < best_score:
+            best_score = score
+            best_item = item
+    
+    if not best_item:
+        return NextItemResult(
+            item=None,
+            selection_method="adaptive",
+            slot=None,
+            level=None,
+            reason="No items with valid IRT parameters available"
+        )
+    
+    return NextItemResult(
+        item=best_item,
+        selection_method="adaptive",
+        slot=best_item.slot,
+        level=best_item.level,
+        reason=f"Adaptive selection - b={best_item.irt_b:.3f} ≈ θ={current_theta:.3f}"
+    )
+
+
+async def get_next_item_hybrid(
+    db: AsyncSession,
+    session_id: str,
+    tryout_id: str,
+    website_id: int,
+    hybrid_transition_slot: int = 10,
+    ai_generation_enabled: bool = False,
+    level_filter: Optional[str] = None
+) -> NextItemResult:
+    """
+    Get next item using hybrid selection.
+    
+    Uses fixed order for first N items, then switches to adaptive.
+    Falls back to CTT if no calibrated items available.
+    
+    Args:
+        db: Database session
+        session_id: Session identifier
+        tryout_id: Tryout identifier
+        website_id: Website identifier
+        hybrid_transition_slot: Slot number to transition from fixed to adaptive
+        ai_generation_enabled: Whether to include AI-generated items
+        level_filter: Optional difficulty level filter
+        
+    Returns:
+        NextItemResult with selected item or None if no items available
+    """
+    # Get session to check current position
+    session_query = select(Session).where(Session.session_id == session_id)
+    session_result = await db.execute(session_query)
+    session = session_result.scalar_one_or_none()
+    
+    if not session:
+        raise CATSelectionError(f"Session {session_id} not found")
+    
+    # Count answered items to determine current position
+    count_query = select(func.count(UserAnswer.id)).where(
+        UserAnswer.session_id == session_id
+    )
+    count_result = await db.execute(count_query)
+    items_answered = count_result.scalar() or 0
+    
+    # Determine current slot (next slot to fill)
+    current_slot = items_answered + 1
+    
+    # Check if we're still in fixed phase
+    if current_slot <= hybrid_transition_slot:
+        # Use fixed selection for initial items
+        result = await get_next_item_fixed(
+            db, session_id, tryout_id, website_id, level_filter
+        )
+        result.selection_method = "hybrid_fixed"
+        result.reason = f"Hybrid mode (fixed phase) - slot {current_slot}"
+        return result
+    
+    # Try adaptive selection
+    adaptive_result = await get_next_item_adaptive(
+        db, session_id, tryout_id, website_id, ai_generation_enabled, level_filter
+    )
+    
+    if adaptive_result.item is not None:
+        adaptive_result.selection_method = "hybrid_adaptive"
+        adaptive_result.reason = f"Hybrid mode (adaptive phase) - {adaptive_result.reason}"
+        return adaptive_result
+    
+    # Fallback to fixed selection if no calibrated items available
+    fixed_result = await get_next_item_fixed(
+        db, session_id, tryout_id, website_id, level_filter
+    )
+    fixed_result.selection_method = "hybrid_fallback"
+    fixed_result.reason = f"Hybrid mode (CTT fallback) - {fixed_result.reason}"
+    return fixed_result
+
+
+async def update_theta(
+    db: AsyncSession,
+    session_id: str,
+    item_id: int,
+    is_correct: bool
+) -> tuple[float, float]:
+    """
+    Update session theta estimate based on response.
+    
+    Calls estimate_theta from irt_calibration.py.
+    Updates session.theta and session.theta_se.
+    Handles initial theta (uses 0.0 for first item).
+    Clamps theta to [-3, +3].
+    
+    Args:
+        db: Database session
+        session_id: Session identifier
+        item_id: Item that was answered
+        is_correct: Whether the answer was correct
+        
+    Returns:
+        Tuple of (theta, theta_se)
+    """
+    return await update_theta_after_response(db, session_id, item_id, is_correct)
+
+
+async def should_terminate(
+    db: AsyncSession,
+    session_id: str,
+    max_items: Optional[int] = None,
+    se_threshold: float = DEFAULT_SE_THRESHOLD
+) -> TerminationCheck:
+    """
+    Check if session should terminate.
+    
+    Termination conditions:
+    - Reached max_items
+    - Reached SE threshold (theta_se < se_threshold)
+    - No more items available
+    
+    Args:
+        db: Database session
+        session_id: Session identifier
+        max_items: Maximum items allowed (None = no limit)
+        se_threshold: SE threshold for termination
+        
+    Returns:
+        TerminationCheck with termination status and reason
+    """
+    # Get session
+    session_query = select(Session).where(Session.session_id == session_id)
+    session_result = await db.execute(session_query)
+    session = session_result.scalar_one_or_none()
+    
+    if not session:
+        raise CATSelectionError(f"Session {session_id} not found")
+    
+    # Count answered items
+    count_query = select(func.count(UserAnswer.id)).where(
+        UserAnswer.session_id == session_id
+    )
+    count_result = await db.execute(count_query)
+    items_answered = count_result.scalar() or 0
+    
+    # Check max items
+    max_items_reached = False
+    if max_items is not None and items_answered >= max_items:
+        max_items_reached = True
+    
+    # Check SE threshold
+    current_se = session.theta_se
+    se_threshold_met = False
+    if current_se is not None and current_se < se_threshold:
+        se_threshold_met = True
+    
+    # Check if we have enough items for SE threshold (at least 15 items per PRD)
+    min_items_for_se = 15
+    se_threshold_met = se_threshold_met and items_answered >= min_items_for_se
+    
+    # Determine termination
+    should_term = max_items_reached or se_threshold_met
+    
+    # Build reason
+    reasons = []
+    if max_items_reached:
+        reasons.append(f"max items reached ({items_answered}/{max_items})")
+    if se_threshold_met:
+        reasons.append(f"SE threshold met ({current_se:.3f} < {se_threshold})")
+    
+    if not reasons:
+        reasons.append("continuing")
+    
+    return TerminationCheck(
+        should_terminate=should_term,
+        reason="; ".join(reasons),
+        items_answered=items_answered,
+        current_se=current_se,
+        max_items=max_items,
+        se_threshold_met=se_threshold_met
+    )
+
+
+async def get_next_item(
+    db: AsyncSession,
+    session_id: str,
+    selection_mode: Literal["fixed", "adaptive", "hybrid"] = "fixed",
+    hybrid_transition_slot: int = 10,
+    ai_generation_enabled: bool = False,
+    level_filter: Optional[str] = None
+) -> NextItemResult:
+    """
+    Get next item based on selection mode.
+    
+    Main entry point for item selection.
+    
+    Args:
+        db: Database session
+        session_id: Session identifier
+        selection_mode: Selection mode ('fixed', 'adaptive', 'hybrid')
+        hybrid_transition_slot: Slot to transition in hybrid mode
+        ai_generation_enabled: Whether AI generation is enabled
+        level_filter: Optional difficulty level filter
+        
+    Returns:
+        NextItemResult with selected item
+    """
+    # Get session for tryout info
+    session_query = select(Session).where(Session.session_id == session_id)
+    session_result = await db.execute(session_query)
+    session = session_result.scalar_one_or_none()
+    
+    if not session:
+        raise CATSelectionError(f"Session {session_id} not found")
+    
+    tryout_id = session.tryout_id
+    website_id = session.website_id
+    
+    if selection_mode == "fixed":
+        return await get_next_item_fixed(
+            db, session_id, tryout_id, website_id, level_filter
+        )
+    elif selection_mode == "adaptive":
+        return await get_next_item_adaptive(
+            db, session_id, tryout_id, website_id, ai_generation_enabled, level_filter
+        )
+    elif selection_mode == "hybrid":
+        return await get_next_item_hybrid(
+            db, session_id, tryout_id, website_id,
+            hybrid_transition_slot, ai_generation_enabled, level_filter
+        )
+    else:
+        raise CATSelectionError(f"Unknown selection mode: {selection_mode}")
+
+
+async def check_user_level_reuse(
+    db: AsyncSession,
+    wp_user_id: str,
+    website_id: int,
+    tryout_id: str,
+    slot: int,
+    level: str
+) -> bool:
+    """
+    Check if user has already answered a question at this difficulty level.
+    
+    Per PRD FR-5.3: Check if student user_id already answered question
+    at specific difficulty level.
+    
+    Args:
+        db: Database session
+        wp_user_id: WordPress user ID
+        website_id: Website identifier
+        tryout_id: Tryout identifier
+        slot: Question slot
+        level: Difficulty level
+        
+    Returns:
+        True if user has answered at this level, False otherwise
+    """
+    # Check if user has answered any item at this slot/level combination
+    query = (
+        select(func.count(UserAnswer.id))
+        .join(Item, UserAnswer.item_id == Item.id)
+        .where(
+            UserAnswer.wp_user_id == wp_user_id,
+            UserAnswer.website_id == website_id,
+            UserAnswer.tryout_id == tryout_id,
+            Item.slot == slot,
+            Item.level == level
+        )
+    )
+    
+    result = await db.execute(query)
+    count = result.scalar() or 0
+    
+    return count > 0
+
+
+async def get_available_levels_for_slot(
+    db: AsyncSession,
+    tryout_id: str,
+    website_id: int,
+    slot: int
+) -> list[str]:
+    """
+    Get available difficulty levels for a specific slot.
+    
+    Args:
+        db: Database session
+        tryout_id: Tryout identifier
+        website_id: Website identifier
+        slot: Question slot
+        
+    Returns:
+        List of available levels
+    """
+    query = (
+        select(Item.level)
+        .where(
+            Item.tryout_id == tryout_id,
+            Item.website_id == website_id,
+            Item.slot == slot
+        )
+        .distinct()
+    )
+    
+    result = await db.execute(query)
+    levels = [row[0] for row in result.all()]
+    
+    return levels
+
+
+# Admin playground functions for testing CAT behavior
+
+async def simulate_cat_selection(
+    db: AsyncSession,
+    tryout_id: str,
+    website_id: int,
+    initial_theta: float = 0.0,
+    selection_mode: Literal["fixed", "adaptive", "hybrid"] = "adaptive",
+    max_items: int = 15,
+    se_threshold: float = DEFAULT_SE_THRESHOLD,
+    hybrid_transition_slot: int = 10
+) -> dict:
+    """
+    Simulate CAT selection for admin testing.
+    
+    Returns sequence of selected items with b values and theta progression.
+    
+    Args:
+        db: Database session
+        tryout_id: Tryout identifier
+        website_id: Website identifier
+        initial_theta: Starting theta value
+        selection_mode: Selection mode to use
+        max_items: Maximum items to simulate
+        se_threshold: SE threshold for termination
+        hybrid_transition_slot: Slot to transition in hybrid mode
+        
+    Returns:
+        Dict with simulation results
+    """
+    # Get all items for this tryout
+    items_query = (
+        select(Item)
+        .where(
+            Item.tryout_id == tryout_id,
+            Item.website_id == website_id
+        )
+        .order_by(Item.slot)
+    )
+    
+    items_result = await db.execute(items_query)
+    all_items = list(items_result.scalars().all())
+    
+    if not all_items:
+        return {
+            "error": "No items found for this tryout",
+            "tryout_id": tryout_id,
+            "website_id": website_id
+        }
+    
+    # Simulate selection
+    selected_items = []
+    current_theta = initial_theta
+    current_se = 3.0  # Start with high uncertainty
+    used_item_ids = set()
+    
+    for i in range(max_items):
+        # Get available items
+        available_items = [item for item in all_items if item.id not in used_item_ids]
+        
+        if not available_items:
+            break
+        
+        # Select based on mode
+        if selection_mode == "adaptive":
+            # Filter to calibrated items only
+            calibrated_items = [item for item in available_items if item.calibrated and item.irt_b is not None]
+            
+            if not calibrated_items:
+                # Fallback to any available item
+                calibrated_items = available_items
+            
+            # Find item closest to current theta
+            best_item = min(
+                calibrated_items,
+                key=lambda item: abs((item.irt_b or 0) - current_theta)
+            )
+        elif selection_mode == "fixed":
+            # Select in slot order
+            best_item = min(available_items, key=lambda item: item.slot)
+        else:  # hybrid
+            if i < hybrid_transition_slot:
+                best_item = min(available_items, key=lambda item: item.slot)
+            else:
+                calibrated_items = [item for item in available_items if item.calibrated and item.irt_b is not None]
+                if calibrated_items:
+                    best_item = min(
+                        calibrated_items,
+                        key=lambda item: abs((item.irt_b or 0) - current_theta)
+                    )
+                else:
+                    best_item = min(available_items, key=lambda item: item.slot)
+        
+        used_item_ids.add(best_item.id)
+        
+        # Simulate response (random based on probability)
+        import random
+        b = best_item.irt_b or estimate_b_from_ctt_p(best_item.ctt_p) if best_item.ctt_p else 0.0
+        p_correct = 1.0 / (1.0 + math.exp(-(current_theta - b)))
+        is_correct = random.random() < p_correct
+        
+        # Update theta (simplified)
+        responses = [1 if item.get('is_correct', True) else 0 for item in selected_items]
+        responses.append(1 if is_correct else 0)
+        b_params = [item['b'] for item in selected_items]
+        b_params.append(b)
+        
+        new_theta, new_se = estimate_theta_mle(responses, b_params, current_theta)
+        current_theta = new_theta
+        current_se = new_se
+        
+        selected_items.append({
+            "slot": best_item.slot,
+            "level": best_item.level,
+            "b": b,
+            "is_correct": is_correct,
+            "theta_after": current_theta,
+            "se_after": current_se,
+            "calibrated": best_item.calibrated
+        })
+        
+        # Check SE threshold
+        if current_se < se_threshold and i >= 14:  # At least 15 items
+            break
+    
+    return {
+        "tryout_id": tryout_id,
+        "website_id": website_id,
+        "initial_theta": initial_theta,
+        "selection_mode": selection_mode,
+        "total_items": len(selected_items),
+        "final_theta": current_theta,
+        "final_se": current_se,
+        "se_threshold_met": current_se < se_threshold,
+        "items": selected_items
+    }