""" AI Question Generation Service. Handles OpenRouter API integration for generating question variants. Implements caching, user-level reuse checking, and prompt engineering. """ import json import logging import re import ast import time from dataclasses import dataclass from typing import Any, Dict, Literal, Optional, Union import httpx from sqlalchemy import and_, func, select from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import get_settings from app.models.item import Item from app.models.ai_generation_run import AIGenerationRun from app.models.tryout import Tryout from app.models.user_answer import UserAnswer from app.schemas.ai import AIModelPricing, AIUsageInfo, GeneratedQuestion logger = logging.getLogger(__name__) settings = get_settings() # OpenRouter API configuration OPENROUTER_API_URL = "https://openrouter.ai/api/v1/chat/completions" OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" # Supported AI models SUPPORTED_MODELS = { settings.OPENROUTER_MODEL_CHEAP: "Mistral Small 4 (Cheap / Fast)", settings.OPENROUTER_MODEL_QWEN: "Qwen 2.5 32B Instruct (Balanced)", settings.OPENROUTER_MODEL_LLAMA: "Llama 3.3 70B (Premium)", } # Level mapping for prompts LEVEL_DESCRIPTIONS = { "mudah": "easier (simpler concepts, more straightforward calculations)", "sedang": "medium difficulty", "sulit": "harder (more complex concepts, multi-step reasoning)", } OPTION_LABELS = tuple("ABCDEFGHIJKLMNOPQRSTUVWXYZ") MODEL_PRICING_CACHE_TTL_SECONDS = 60 * 30 _model_pricing_cache: dict[str, tuple[float, AIModelPricing | None]] = {} @dataclass class OpenRouterCallResult: content: str usage: AIUsageInfo | None = None def get_option_labels(options: Dict[str, str] | None) -> list[str]: labels = { str(key).strip().upper() for key, value in (options or {}).items() if str(key).strip() and str(value).strip() } return [label for label in OPTION_LABELS if label in labels] def _parse_openrouter_price(value: Any) -> float | None: if value is None: return None try: price = float(value) except (TypeError, ValueError): return None return price if price >= 0 else None def _build_pricing(raw_pricing: dict[str, Any] | None) -> AIModelPricing | None: if not raw_pricing: return None prompt = _parse_openrouter_price(raw_pricing.get("prompt")) completion = _parse_openrouter_price(raw_pricing.get("completion")) if prompt is None and completion is None: return None return AIModelPricing( prompt=prompt, completion=completion, prompt_per_million=prompt * 1_000_000 if prompt is not None else None, completion_per_million=completion * 1_000_000 if completion is not None else None, ) async def get_model_pricing(model_id: str) -> AIModelPricing | None: cached = _model_pricing_cache.get(model_id) now = time.monotonic() if cached and now - cached[0] < MODEL_PRICING_CACHE_TTL_SECONDS: return cached[1] headers = {"Content-Type": "application/json"} if settings.OPENROUTER_API_KEY: headers["Authorization"] = f"Bearer {settings.OPENROUTER_API_KEY}" try: timeout = httpx.Timeout(min(settings.OPENROUTER_TIMEOUT, 5)) async with httpx.AsyncClient(timeout=timeout) as client: response = await client.get(OPENROUTER_MODELS_URL, headers=headers) if response.status_code != 200: logger.warning( "OpenRouter models pricing request failed: %s - %s", response.status_code, response.text[:240], ) _model_pricing_cache[model_id] = (now, None) return None for model in response.json().get("data", []): if model.get("id") == model_id: pricing = _build_pricing(model.get("pricing")) _model_pricing_cache[model_id] = (now, pricing) return pricing except Exception as exc: logger.warning("OpenRouter model pricing lookup failed for %s: %s", model_id, exc) _model_pricing_cache[model_id] = (now, None) return None def _calculate_usage_cost( prompt_tokens: int | None, completion_tokens: int | None, pricing: AIModelPricing | None, provider_cost: Any = None, ) -> float | None: provider_cost_value = _parse_openrouter_price(provider_cost) if provider_cost_value is not None: return provider_cost_value if pricing is None: return None cost = 0.0 has_cost_component = False if prompt_tokens is not None and pricing.prompt is not None: cost += prompt_tokens * pricing.prompt has_cost_component = True if completion_tokens is not None and pricing.completion is not None: cost += completion_tokens * pricing.completion has_cost_component = True return cost if has_cost_component else None async def build_usage_info(raw_usage: dict[str, Any] | None, model_id: str) -> AIUsageInfo | None: if not raw_usage: return None def token_count(key: str) -> int | None: value = raw_usage.get(key) if value is None: return None try: return int(value) except (TypeError, ValueError): return None prompt_tokens = token_count("prompt_tokens") completion_tokens = token_count("completion_tokens") total_tokens = token_count("total_tokens") if total_tokens is None and (prompt_tokens is not None or completion_tokens is not None): total_tokens = (prompt_tokens or 0) + (completion_tokens or 0) pricing = await get_model_pricing(model_id) cost_usd = _calculate_usage_cost( prompt_tokens, completion_tokens, pricing, provider_cost=raw_usage.get("cost"), ) return AIUsageInfo( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, cost_usd=cost_usd, ) def combine_usage(usages: list[AIUsageInfo | None]) -> AIUsageInfo | None: filtered = [usage for usage in usages if usage is not None] if not filtered: return None def summed(field: str) -> int | float | None: values = [getattr(usage, field) for usage in filtered] present = [value for value in values if value is not None] return sum(present) if present else None return AIUsageInfo( prompt_tokens=summed("prompt_tokens"), completion_tokens=summed("completion_tokens"), total_tokens=summed("total_tokens"), cost_usd=summed("cost_usd"), ) def get_prompt_template( basis_stem: str, basis_options: Dict[str, str], basis_correct: str, basis_explanation: Optional[str], target_level: Literal["mudah", "sulit"], operator_notes: Optional[str] = None, ) -> str: """ Generate standardized prompt for AI question generation. Args: basis_stem: The basis question stem basis_options: The basis question options basis_correct: The basis correct answer basis_explanation: The basis explanation target_level: Target difficulty level Returns: Formatted prompt string """ level_desc = LEVEL_DESCRIPTIONS.get(target_level, target_level) option_labels = get_option_labels(basis_options) or ["A", "B", "C", "D"] option_count = len(option_labels) option_label_text = ", ".join(option_labels) example_options = {label: f"Option {label} text" for label in option_labels} options_text = "\n".join( [f" {key}: {value}" for key, value in basis_options.items()] ) explanation_text = ( f"Explanation: {basis_explanation}" if basis_explanation else "Explanation: (not provided)" ) notes_block = "" if operator_notes and operator_notes.strip(): notes_block = f""" ADDITIONAL OPERATOR NOTES: {operator_notes.strip()} Apply these notes as style constraints as long as they do not conflict with correctness. """ prompt = f"""You are an educational content creator specializing in creating assessment questions. Given a "Sedang" (medium difficulty) question, generate a new question at a different difficulty level. BASIS QUESTION (Sedang level): Question: {basis_stem} Options: {options_text} Correct Answer: {basis_correct} {explanation_text} TASK: Generate 1 new question that is {level_desc} than the basis question above. {notes_block} REQUIREMENTS: 1. Keep the SAME topic/subject matter as the basis question 2. Use similar context and terminology 3. Create exactly {option_count} answer options with labels exactly: {option_label_text} 4. Preserve the basis option count and option labels. Do not omit, add, rename, or merge answer options. 5. Only ONE correct answer, and it must be one of: {option_label_text} 6. Include a clear explanation of why the correct answer is correct 7. Make the question noticeably {level_desc} - not just a minor variation 8. Follow and preserve the basis question's inline HTML style. Keep structural and inline tags such as

,
, , , , , , , , and simple inline attributes such as text alignment when the basis uses them. 9. Do not escape HTML tags as text. Return HTML markup in the JSON string values exactly as markup. OUTPUT FORMAT: Return ONLY a valid JSON object with this exact structure (no markdown, no code blocks): {{"stem": "Your question text here", "options": {json.dumps(example_options, ensure_ascii=False)}, "correct": "{option_labels[0]}", "explanation": "Explanation text here"}} Remember: The correct field must be exactly one of: {option_label_text}.""" return prompt def parse_ai_response(response_text: str) -> Optional[GeneratedQuestion]: """ Parse AI response to extract question data. Handles various response formats including JSON code blocks. Args: response_text: Raw AI response text Returns: GeneratedQuestion if parsing successful, None otherwise """ if not response_text: return None cleaned = response_text.strip() candidates = _extract_json_candidates(cleaned) for candidate in candidates: candidate_clean = _sanitize_json_candidate(candidate) parsed = _try_parse_json_like(candidate_clean) if isinstance(parsed, dict): question = validate_and_create_question(parsed) if question: return question logger.warning(f"Failed to parse AI response: {cleaned[:240]}...") return None def validate_and_create_question(data: Dict[str, Any]) -> Optional[GeneratedQuestion]: """ Validate parsed data and create GeneratedQuestion. Args: data: Parsed JSON data Returns: GeneratedQuestion if valid, None otherwise """ stem = str(data.get("stem") or data.get("question") or "").strip() if not stem: logger.warning(f"Missing question stem in AI response: {data.keys()}") return None options = _normalize_options(data.get("options")) if not options: logger.warning("Options cannot be normalized to a labeled option map") return None correct = _normalize_correct_answer( data.get("correct") or data.get("correct_answer") or data.get("answer") ) if correct not in set(options.keys()): logger.warning(f"Invalid correct answer: {correct}") return None return GeneratedQuestion( stem=stem, options=options, correct=correct, explanation=str(data.get("explanation") or data.get("rationale") or "").strip() or None, ) def _extract_json_candidates(text: str) -> list[str]: candidates: list[str] = [] code_blocks = re.findall(r"```(?:json)?\s*([\s\S]*?)\s*```", text) candidates.extend(block.strip() for block in code_blocks if block.strip()) balanced = _extract_first_balanced_object(text) if balanced: candidates.append(balanced) candidates.append(text.strip()) deduped: list[str] = [] seen = set() for candidate in candidates: if candidate and candidate not in seen: deduped.append(candidate) seen.add(candidate) return deduped def _extract_first_balanced_object(text: str) -> str | None: start = text.find("{") if start == -1: return None depth = 0 in_string = False escape_next = False for idx in range(start, len(text)): ch = text[idx] if escape_next: escape_next = False continue if ch == "\\" and in_string: escape_next = True continue if ch == '"': in_string = not in_string continue if in_string: continue if ch == "{": depth += 1 elif ch == "}": depth -= 1 if depth == 0: return text[start: idx + 1] return None def _sanitize_json_candidate(candidate: str) -> str: cleaned = candidate.strip().lstrip("\ufeff") cleaned = cleaned.replace("“", '"').replace("”", '"').replace("’", "'") cleaned = re.sub(r",\s*([}\]])", r"\1", cleaned) return cleaned def _try_parse_json_like(candidate: str) -> Any: try: return json.loads(candidate) except json.JSONDecodeError: pass try: # Fallback for Python-like dict outputs using single quotes. return ast.literal_eval(candidate) except (ValueError, SyntaxError): return None def _normalize_options(raw_options: Any) -> dict[str, str]: if isinstance(raw_options, dict): normalized = {str(k).strip().upper(): str(v).strip() for k, v in raw_options.items()} return {k: normalized[k] for k in OPTION_LABELS if normalized.get(k, "")} if isinstance(raw_options, list): mapped: dict[str, str] = {} for idx, opt in enumerate(raw_options): if isinstance(opt, dict): key = str(opt.get("increment") or opt.get("key") or "").strip().upper() text = str(opt.get("text") or opt.get("label") or opt.get("value") or "").strip() else: key = "" text = str(opt).strip() if not key and idx < len(OPTION_LABELS): key = OPTION_LABELS[idx] if key in OPTION_LABELS and text: mapped[key] = text return mapped return {} def _normalize_correct_answer(raw_correct: Any) -> str: if raw_correct is None: return "" raw_text = str(raw_correct).strip().upper() if raw_text in OPTION_LABELS: return raw_text if raw_text.isdigit(): idx = int(raw_text) if 1 <= idx <= len(OPTION_LABELS): return OPTION_LABELS[idx - 1] if 0 <= idx < len(OPTION_LABELS): return OPTION_LABELS[idx] if raw_text.startswith("OPTION ") and raw_text[-1:] in OPTION_LABELS: return raw_text[-1] return raw_text[:1] def generated_matches_basis_options(generated: GeneratedQuestion, basis_item: Item) -> bool: basis_labels = get_option_labels(basis_item.options) generated_labels = get_option_labels(generated.options) if basis_labels != generated_labels: logger.warning( "Generated option labels do not match basis: basis=%s generated=%s", basis_labels, generated_labels, ) return False if generated.correct not in set(basis_labels): logger.warning( "Generated correct answer %s is outside basis labels %s", generated.correct, basis_labels, ) return False return True async def call_openrouter_api( prompt: str, model: str, max_retries: int = 3, ) -> Optional[OpenRouterCallResult]: """ Call OpenRouter API to generate question. Args: prompt: The prompt to send model: AI model to use max_retries: Maximum retry attempts Returns: OpenRouterCallResult with response text and usage, or None if failed """ if not settings.OPENROUTER_API_KEY: logger.error("OPENROUTER_API_KEY not configured") return None if model not in SUPPORTED_MODELS: logger.error(f"Unsupported AI model: {model}") return None headers = { "Authorization": f"Bearer {settings.OPENROUTER_API_KEY}", "Content-Type": "application/json", "HTTP-Referer": "https://github.com/irt-bank-soal", "X-Title": "IRT Bank Soal", } payload: dict[str, Any] = { "model": model, "messages": [ { "role": "user", "content": prompt, } ], "max_tokens": 2000, "temperature": 0.7, } provider_order = [ provider for provider in settings.OPENROUTER_PROVIDER_ORDER if provider.strip() ] if provider_order: payload["provider"] = { "order": provider_order, "allow_fallbacks": settings.OPENROUTER_ALLOW_PROVIDER_FALLBACKS, } timeout = httpx.Timeout(settings.OPENROUTER_TIMEOUT) for attempt in range(max_retries): try: async with httpx.AsyncClient(timeout=timeout) as client: response = await client.post( OPENROUTER_API_URL, headers=headers, json=payload, ) if response.status_code == 200: data = response.json() choices = data.get("choices", []) if choices: message = choices[0].get("message", {}) content = message.get("content") if not content: logger.warning("OpenRouter response had no message content") return None usage = await build_usage_info(data.get("usage"), model) return OpenRouterCallResult(content=content, usage=usage) logger.warning("No choices in OpenRouter response") return None elif response.status_code == 429: # Rate limited - wait and retry logger.warning(f"Rate limited, attempt {attempt + 1}/{max_retries}") if attempt < max_retries - 1: import asyncio await asyncio.sleep(2 ** attempt) continue return None else: logger.error( f"OpenRouter API error: {response.status_code} - {response.text}" ) return None except httpx.TimeoutException: logger.warning(f"OpenRouter timeout, attempt {attempt + 1}/{max_retries}") if attempt < max_retries - 1: continue return None except Exception as e: logger.error(f"OpenRouter API call failed: {e}") if attempt < max_retries - 1: continue return None return None async def generate_question( basis_item: Item, target_level: Literal["mudah", "sulit"], ai_model: str = settings.OPENROUTER_MODEL_QWEN, operator_notes: Optional[str] = None, ) -> Optional[GeneratedQuestion]: """ Generate a new question based on a basis item. Args: basis_item: The basis item (must be sedang level) target_level: Target difficulty level ai_model: AI model to use Returns: GeneratedQuestion if successful, None otherwise """ # Build prompt prompt = get_prompt_template( basis_stem=basis_item.stem, basis_options=basis_item.options, basis_correct=basis_item.correct_answer, basis_explanation=basis_item.explanation, target_level=target_level, operator_notes=operator_notes, ) max_generation_attempts = 3 for attempt in range(1, max_generation_attempts + 1): api_result = await call_openrouter_api(prompt, ai_model) if not api_result: logger.error("No response from OpenRouter API") continue generated = parse_ai_response(api_result.content) if generated and generated_matches_basis_options(generated, basis_item): generated = generated.model_copy(update={"usage": api_result.usage}) return generated logger.warning( "Failed to parse or validate AI response (attempt %s/%s), retrying", attempt, max_generation_attempts, ) return None async def check_cache_reuse( tryout_id: str, slot: int, level: str, wp_user_id: str, website_id: int, db: AsyncSession, ) -> Optional[Item]: """ Check if there's a cached item that the user hasn't answered yet. Query DB for existing item matching (tryout_id, slot, level). Check if user already answered this item at this difficulty level. Args: tryout_id: Tryout identifier slot: Question slot level: Difficulty level wp_user_id: WordPress user ID website_id: Website identifier db: Database session Returns: Cached item if found and user hasn't answered, None otherwise """ # Find existing items at this slot/level result = await db.execute( select(Item).where( and_( Item.tryout_id == tryout_id, Item.website_id == website_id, Item.slot == slot, Item.level == level, ) ) ) existing_items = result.scalars().all() if not existing_items: return None # Check each item to find one the user hasn't answered for item in existing_items: # Check if user has answered this item answer_result = await db.execute( select(UserAnswer).where( and_( UserAnswer.item_id == item.id, UserAnswer.wp_user_id == wp_user_id, ) ) ) user_answer = answer_result.scalar_one_or_none() if user_answer is None: # User hasn't answered this item - can reuse logger.info( f"Cache hit for tryout={tryout_id}, slot={slot}, level={level}, " f"item_id={item.id}, user={wp_user_id}" ) return item # All items have been answered by this user logger.info( f"Cache miss (user answered all) for tryout={tryout_id}, slot={slot}, " f"level={level}, user={wp_user_id}" ) return None async def generate_with_cache_check( tryout_id: str, slot: int, level: Literal["mudah", "sulit"], wp_user_id: str, website_id: int, db: AsyncSession, ai_model: str = settings.OPENROUTER_MODEL_QWEN, ) -> tuple[Optional[Union[Item, GeneratedQuestion]], bool]: """ Generate question with cache checking. First checks if AI generation is enabled for the tryout. Then checks for cached items the user hasn't answered. If cache miss, generates new question via AI. Args: tryout_id: Tryout identifier slot: Question slot level: Target difficulty level wp_user_id: WordPress user ID website_id: Website identifier db: Database session ai_model: AI model to use Returns: Tuple of (item/question or None, is_cached) """ # Check if AI generation is enabled for this tryout tryout_result = await db.execute( select(Tryout).where( and_( Tryout.tryout_id == tryout_id, Tryout.website_id == website_id, ) ) ) tryout = tryout_result.scalar_one_or_none() if tryout and not tryout.ai_generation_enabled: logger.info(f"AI generation disabled for tryout={tryout_id}") # Still check cache even if AI disabled cached_item = await check_cache_reuse( tryout_id, slot, level, wp_user_id, website_id, db ) if cached_item: return cached_item, True return None, False # Check cache for reusable item cached_item = await check_cache_reuse( tryout_id, slot, level, wp_user_id, website_id, db ) if cached_item: return cached_item, True # Cache miss - need to generate # Get basis item (sedang level at same slot) basis_result = await db.execute( select(Item).where( and_( Item.tryout_id == tryout_id, Item.website_id == website_id, Item.slot == slot, Item.level == "sedang", ) ).limit(1) ) basis_item = basis_result.scalar_one_or_none() if not basis_item: logger.error( f"No basis item found for tryout={tryout_id}, slot={slot}" ) return None, False # Generate new question generated = await generate_question(basis_item, level, ai_model) if not generated: logger.error( f"Failed to generate question for tryout={tryout_id}, slot={slot}, level={level}" ) return None, False return generated, False async def save_ai_question( generated_data: GeneratedQuestion, tryout_id: str, website_id: int, basis_item_id: int, slot: int, level: Literal["mudah", "sedang", "sulit"], ai_model: str, db: AsyncSession, generation_run_id: int | None = None, source_snapshot_question_id: int | None = None, variant_status: str = "draft", ) -> Optional[int]: """ Save AI-generated question to database. Args: generated_data: Generated question data tryout_id: Tryout identifier website_id: Website identifier basis_item_id: Basis item ID slot: Question slot level: Difficulty level ai_model: AI model used db: Database session Returns: Created item ID or None if failed """ try: new_item = Item( tryout_id=tryout_id, website_id=website_id, slot=slot, level=level, stem=generated_data.stem, options=generated_data.options, correct_answer=generated_data.correct, explanation=generated_data.explanation, generated_by="ai", ai_model=ai_model, basis_item_id=basis_item_id, generation_run_id=generation_run_id, source_snapshot_question_id=source_snapshot_question_id, variant_status=variant_status, calibrated=False, ctt_p=None, ctt_bobot=None, ctt_category=None, irt_b=None, irt_se=None, calibration_sample_size=0, ) db.add(new_item) await db.flush() # Get the ID without committing logger.info( f"Saved AI-generated item: id={new_item.id}, tryout={tryout_id}, " f"slot={slot}, level={level}, model={ai_model}" ) return new_item.id except Exception as e: logger.error(f"Failed to save AI-generated question: {e}") return None async def create_generation_run( basis_item_id: int, target_level: Literal["mudah", "sulit"], requested_count: int, model: str, created_by: str, db: AsyncSession, source_snapshot_question_id: int | None = None, operator_notes: str | None = None, prompt_version: str = "v1", ) -> int: run = AIGenerationRun( basis_item_id=basis_item_id, source_snapshot_question_id=source_snapshot_question_id, target_level=target_level, requested_count=requested_count, model=model, prompt_version=prompt_version, operator_notes=operator_notes, created_by=created_by, ) db.add(run) await db.flush() return int(run.id) async def generate_questions_batch( basis_item: Item, target_level: Literal["mudah", "sulit"], ai_model: str, count: int, operator_notes: Optional[str] = None, ) -> list[GeneratedQuestion]: generated_items: list[GeneratedQuestion] = [] for _ in range(count): generated = await generate_question( basis_item=basis_item, target_level=target_level, ai_model=ai_model, operator_notes=operator_notes, ) if generated is not None: generated_items.append(generated) return generated_items async def get_ai_stats(db: AsyncSession, website_id: int | None = None) -> Dict[str, Any]: """ Get AI generation statistics. Args: db: Database session Returns: Statistics dictionary """ filters = [Item.generated_by == "ai"] if website_id is not None: filters.append(Item.website_id == website_id) # Total AI-generated items total_result = await db.execute(select(func.count(Item.id)).where(*filters)) total_ai_items = total_result.scalar() or 0 # Items by model model_result = await db.execute( select(Item.ai_model, func.count(Item.id)) .where(*filters) .where(Item.ai_model.isnot(None)) .group_by(Item.ai_model) ) items_by_model = {row[0]: row[1] for row in model_result.all()} # Note: Cache hit rate would need to be tracked separately # This is a placeholder for now return { "total_ai_items": total_ai_items, "items_by_model": items_by_model, "cache_hit_rate": 0.0, "total_cache_hits": 0, "total_requests": 0, } def validate_ai_model(model: str) -> bool: """ Validate that the AI model is supported. Args: model: AI model identifier Returns: True if model is supported """ return model in SUPPORTED_MODELS