diff --git a/alembic/versions/20260404_000003_ai_runs_and_variant_status.py b/alembic/versions/20260404_000003_ai_runs_and_variant_status.py new file mode 100644 index 0000000..75699c9 --- /dev/null +++ b/alembic/versions/20260404_000003_ai_runs_and_variant_status.py @@ -0,0 +1,118 @@ +"""add ai generation runs and item variant lifecycle fields + +Revision ID: 20260404_000003 +Revises: 20260402_000002 +Create Date: 2026-04-04 10:10:00 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260404_000003" +down_revision: Union[str, None] = "20260402_000002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "ai_generation_runs", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("basis_item_id", sa.Integer(), nullable=False), + sa.Column("source_snapshot_question_id", sa.Integer(), nullable=True), + sa.Column("target_level", sa.String(length=50), nullable=False), + sa.Column("requested_count", sa.Integer(), nullable=False, server_default="1"), + sa.Column("model", sa.String(length=255), nullable=False), + sa.Column("prompt_version", sa.String(length=50), nullable=False, server_default="v1"), + sa.Column("operator_notes", sa.Text(), nullable=True), + sa.Column("created_by", sa.String(length=255), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["basis_item_id"], ["items.id"], ondelete="CASCADE", onupdate="CASCADE"), + sa.ForeignKeyConstraint( + ["source_snapshot_question_id"], + ["tryout_snapshot_questions.id"], + ondelete="SET NULL", + onupdate="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_ai_generation_runs_basis_item_id", "ai_generation_runs", ["basis_item_id"], unique=False) + op.create_index( + "ix_ai_generation_runs_source_snapshot_question_id", + "ai_generation_runs", + ["source_snapshot_question_id"], + unique=False, + ) + + op.add_column("items", sa.Column("generation_run_id", sa.Integer(), nullable=True)) + op.add_column("items", sa.Column("source_snapshot_question_id", sa.Integer(), nullable=True)) + op.add_column("items", sa.Column("variant_status", sa.String(length=50), nullable=False, server_default="active")) + op.add_column("items", sa.Column("reviewed_by", sa.String(length=255), nullable=True)) + op.add_column("items", sa.Column("reviewed_at", sa.DateTime(timezone=True), nullable=True)) + op.add_column("items", sa.Column("review_notes", sa.Text(), nullable=True)) + + op.create_foreign_key( + "fk_items_generation_run_id", + "items", + "ai_generation_runs", + ["generation_run_id"], + ["id"], + ondelete="SET NULL", + onupdate="CASCADE", + ) + op.create_foreign_key( + "fk_items_source_snapshot_question_id", + "items", + "tryout_snapshot_questions", + ["source_snapshot_question_id"], + ["id"], + ondelete="SET NULL", + onupdate="CASCADE", + ) + op.create_index("ix_items_generation_run_id", "items", ["generation_run_id"], unique=False) + op.create_index( + "ix_items_source_snapshot_question_id", + "items", + ["source_snapshot_question_id"], + unique=False, + ) + op.create_index("ix_items_variant_status", "items", ["variant_status"], unique=False) + + op.drop_index("ix_items_tryout_id_website_id_slot", table_name="items") + op.create_index( + "ix_items_tryout_id_website_id_slot", + "items", + ["tryout_id", "website_id", "slot", "level"], + unique=False, + ) + + op.alter_column("items", "variant_status", server_default=None) + + +def downgrade() -> None: + op.drop_index("ix_items_tryout_id_website_id_slot", table_name="items") + op.create_index( + "ix_items_tryout_id_website_id_slot", + "items", + ["tryout_id", "website_id", "slot", "level"], + unique=True, + ) + + op.drop_index("ix_items_variant_status", table_name="items") + op.drop_index("ix_items_source_snapshot_question_id", table_name="items") + op.drop_index("ix_items_generation_run_id", table_name="items") + op.drop_constraint("fk_items_source_snapshot_question_id", "items", type_="foreignkey") + op.drop_constraint("fk_items_generation_run_id", "items", type_="foreignkey") + op.drop_column("items", "review_notes") + op.drop_column("items", "reviewed_at") + op.drop_column("items", "reviewed_by") + op.drop_column("items", "variant_status") + op.drop_column("items", "source_snapshot_question_id") + op.drop_column("items", "generation_run_id") + + op.drop_index("ix_ai_generation_runs_source_snapshot_question_id", table_name="ai_generation_runs") + op.drop_index("ix_ai_generation_runs_basis_item_id", table_name="ai_generation_runs") + op.drop_table("ai_generation_runs") diff --git a/app/admin_web.py b/app/admin_web.py index 4efc069..2359d20 100644 --- a/app/admin_web.py +++ b/app/admin_web.py @@ -8,6 +8,7 @@ Tortoise-oriented internals that do not match this project. import secrets import uuid from dataclasses import dataclass +from datetime import datetime, timezone from html import escape import json from typing import Any @@ -22,10 +23,20 @@ from starlette.status import HTTP_303_SEE_OTHER, HTTP_401_UNAUTHORIZED from app.core.config import get_settings from app.database import get_db -from app.models import Item, Session, Tryout, TryoutImportSnapshot, TryoutSnapshotQuestion, Website +from app.models import ( + AIGenerationRun, + Item, + Session, + Tryout, + TryoutImportSnapshot, + TryoutSnapshotQuestion, + UserAnswer, + Website, +) from app.services.ai_generation import ( SUPPORTED_MODELS, - generate_question, + create_generation_run, + generate_questions_batch, get_ai_stats, save_ai_question, validate_ai_model, @@ -182,6 +193,7 @@ def _render_admin_page(title: str, page_title: str, body: str) -> HTMLResponse: Dashboard Websites Tryout Import + Basis Items Calibration Status Item Statistics Session Overview @@ -493,6 +505,249 @@ async def _basis_items_for_playground(db: AsyncSession, limit: int = 20) -> list return list(result.scalars().all()) +async def _recent_generation_runs(db: AsyncSession, limit: int = 20) -> list[AIGenerationRun]: + result = await db.execute( + select(AIGenerationRun).order_by(AIGenerationRun.id.desc()).limit(limit) + ) + return list(result.scalars().all()) + + +async def _recent_generated_variants( + db: AsyncSession, + limit: int = 100, + basis_item_id: int | None = None, +) -> list[Item]: + stmt = select(Item).where(Item.generated_by == "ai") + if basis_item_id is not None: + stmt = stmt.where(Item.basis_item_id == basis_item_id) + result = await db.execute( + stmt.order_by(Item.created_at.desc(), Item.id.desc()).limit(limit) + ) + return list(result.scalars().all()) + + +async def _usage_metrics_for_items( + db: AsyncSession, + item_ids: list[int], +) -> dict[int, dict[str, float]]: + if not item_ids: + return {} + + result = await db.execute( + select( + UserAnswer.item_id, + func.count(UserAnswer.id).label("impressions"), + func.count(func.distinct(UserAnswer.wp_user_id)).label("unique_users"), + ) + .where(UserAnswer.item_id.in_(item_ids)) + .group_by(UserAnswer.item_id) + ) + + metrics: dict[int, dict[str, float]] = {} + for item_id, impressions, unique_users in result.all(): + impressions_i = int(impressions or 0) + unique_users_i = int(unique_users or 0) + frequency = (impressions_i / unique_users_i) if unique_users_i else 0.0 + metrics[int(item_id)] = { + "impressions": float(impressions_i), + "unique_users": float(unique_users_i), + "frequency": float(frequency), + } + return metrics + + +async def _family_usage_stats( + db: AsyncSession, + basis_item: Item, + variants: list[Item], +) -> tuple[dict[int, dict[str, float]], dict[str, float]]: + family_item_ids = [basis_item.id] + [item.id for item in variants] + usage_metrics = await _usage_metrics_for_items(db, family_item_ids) + family_impressions = int(sum(metric["impressions"] for metric in usage_metrics.values())) + family_unique_users = int( + await db.scalar( + select(func.count(func.distinct(UserAnswer.wp_user_id))).where( + UserAnswer.item_id.in_(family_item_ids) + ) + ) + or 0 + ) + family_frequency = (family_impressions / family_unique_users) if family_unique_users else 0.0 + return usage_metrics, { + "impressions": float(family_impressions), + "unique_users": float(family_unique_users), + "frequency": float(family_frequency), + } + + +def _basis_items_list_body(items: list[Item]) -> str: + rows = [] + for item in items: + rows.append( + "" + f"{item.id}" + f"{escape(item.tryout_id)}" + f"{item.slot}" + f"{item.website_id}" + f"{escape(_truncate(item.stem, 120))}" + f"{item.source_snapshot_question_id or '-'}" + f"Open Workspace" + "" + ) + table = ( + "" + + ("".join(rows) if rows else "") + + "
Item IDTryoutSlotWebsiteStemSource Snapshot QIDActions
No basis items found.
" + ) + return f""" +

Basis items are canonical parent questions (sedang, non-AI). Open a workspace to generate and review AI child variants.

+ {table} + """ + + +def _basis_item_workspace_body( + basis_item: Item, + runs: list[AIGenerationRun], + variants: list[Item], + usage_by_item: dict[int, dict[str, float]], + family_stats: dict[str, float], + filters: dict[str, str], + error: str | None = None, + success: str | None = None, + target_level: str = "mudah", + ai_model: str = settings.OPENROUTER_MODEL_QWEN, + generation_count: str = "1", + operator_notes: str = "", +) -> str: + error_html = f'
{escape(error)}
' if error else "" + success_html = f'
{escape(success)}
' if success else "" + model_options = "".join( + f'' + for model, label in SUPPORTED_MODELS.items() + ) + status_filter = filters.get("status", "") + level_filter = filters.get("level", "") + min_frequency_filter = filters.get("min_frequency", "") + run_id_filter = filters.get("run_id", "") + + run_rows = [ + [ + run.id, + run.target_level, + run.requested_count, + run.model, + run.created_by, + str(run.created_at), + ] + for run in runs + ] + runs_table = _table( + ["Run ID", "Target", "Requested", "Model", "Created By", "Created At"], + run_rows, + ) + + variant_rows = [] + for item in variants: + usage = usage_by_item.get(item.id, {"impressions": 0.0, "unique_users": 0.0, "frequency": 0.0}) + variant_rows.append( + "" + f"" + f"{item.id}" + f"{item.generation_run_id or '-'}" + f"{escape(item.level)}" + f"{escape(item.variant_status)}" + f"{escape(item.ai_model or '-')}" + f"{int(usage['impressions'])}" + f"{int(usage['unique_users'])}" + f"{usage['frequency']:.2f}" + f"{escape(_truncate(item.stem, 130))}" + f"{escape(str(item.created_at))}" + "" + ) + variants_table = ( + f"
" + "
" + "" + "" + "
" + "" + + ("".join(variant_rows) if variant_rows else "") + + "
el.checked = this.checked)\">Item IDRun IDLevelStatusModelImpressionsUnique UsersFrequencyStemCreated At
No generated variants yet for this parent.
" + ) + + return f""" +

+ Parent Item: #{basis_item.id} | + Tryout: {escape(basis_item.tryout_id)} | + Slot: {basis_item.slot} | + Website: {basis_item.website_id} | + Source Snapshot QID: {basis_item.source_snapshot_question_id or '-'} +

+

+ Family Usage: impressions={int(family_stats.get("impressions", 0.0))}, + unique users={int(family_stats.get("unique_users", 0.0))}, + frequency={family_stats.get("frequency", 0.0):.2f} +

+

Stem: {escape(_truncate(basis_item.stem, 260))}

+ {success_html} + {error_html} +

Variant Filters

+
+ + + + + + + + +
+ + Reset +
+
+
+ + + + + + +

Recommended: 1-3 per run. Larger runs increase overlap and review burden.

+ + + +
+

Generation Runs for This Parent

+ {runs_table} +

Child Variants for This Parent

+

Filtered variants shown: {len(variants)}

+ {variants_table} +

Back to Basis Items

+ """ + + async def _find_or_create_demo_basis_item(db: AsyncSession) -> Item: result = await db.execute( select(Item) @@ -680,6 +935,8 @@ async def _promote_snapshot_question_to_item( correct_answer=question.correct_answer, explanation=question.explanation_html, generated_by="manual", + source_snapshot_question_id=question.id, + variant_status="active", calibrated=False, calibration_sample_size=0, ) @@ -1382,16 +1639,320 @@ async def session_overview_view(request: Request, db: AsyncSession = Depends(get return _render_admin_page("Session Overview", "Session Overview", body) +@router.get("/basis-items", include_in_schema=False) +async def basis_items_view(request: Request, db: AsyncSession = Depends(get_db)): + admin = await _current_admin(request) + if not admin: + return _login_redirect() + + result = await db.execute( + select(Item) + .where(Item.level == "sedang", Item.generated_by != "ai") + .order_by(Item.updated_at.desc(), Item.id.desc()) + .limit(200) + ) + basis_items = list(result.scalars().all()) + body = _basis_items_list_body(basis_items) + return _render_admin_page("Basis Items", "Basis Items", body) + + +@router.get("/basis-items/{basis_item_id}", include_in_schema=False) +async def basis_item_workspace_view( + basis_item_id: int, + request: Request, + db: AsyncSession = Depends(get_db), +): + admin = await _current_admin(request) + if not admin: + return _login_redirect() + + status_filter = (request.query_params.get("status") or "").strip() + level_filter = (request.query_params.get("level") or "").strip() + run_id_filter = (request.query_params.get("run_id") or "").strip() + min_frequency_filter = (request.query_params.get("min_frequency") or "").strip() + filters = { + "status": status_filter, + "level": level_filter, + "run_id": run_id_filter, + "min_frequency": min_frequency_filter, + } + + basis_item = await db.get(Item, basis_item_id) + if basis_item is None or basis_item.generated_by == "ai" or basis_item.level != "sedang": + result = await db.execute( + select(Item) + .where(Item.level == "sedang", Item.generated_by != "ai") + .order_by(Item.updated_at.desc(), Item.id.desc()) + .limit(200) + ) + body = _basis_items_list_body(list(result.scalars().all())) + return _render_admin_page("Basis Items", "Basis Items", body) + + run_result = await db.execute( + select(AIGenerationRun) + .where(AIGenerationRun.basis_item_id == basis_item.id) + .order_by(AIGenerationRun.id.desc()) + .limit(50) + ) + runs = list(run_result.scalars().all()) + variant_result = await db.execute( + select(Item).where(Item.generated_by == "ai", Item.basis_item_id == basis_item.id) + .order_by(Item.created_at.desc(), Item.id.desc()) + .limit(300) + ) + variants_all = list(variant_result.scalars().all()) + variants = variants_all + if status_filter: + variants = [item for item in variants if item.variant_status == status_filter] + if level_filter in {"mudah", "sulit"}: + variants = [item for item in variants if item.level == level_filter] + if run_id_filter.isdigit(): + rid = int(run_id_filter) + variants = [item for item in variants if item.generation_run_id == rid] + usage_metrics, family_stats = await _family_usage_stats(db, basis_item, variants) + if min_frequency_filter: + try: + min_freq = float(min_frequency_filter) + variants = [ + item + for item in variants + if usage_metrics.get(item.id, {}).get("frequency", 0.0) >= min_freq + ] + except ValueError: + pass + body = _basis_item_workspace_body( + basis_item, + runs, + variants, + usage_metrics, + family_stats, + filters, + ) + return _render_admin_page( + f"Basis Item #{basis_item.id}", + f"Basis Item Workspace #{basis_item.id}", + body, + ) + + +@router.post("/basis-items/{basis_item_id}/generate", include_in_schema=False) +async def basis_item_generate_submit( + basis_item_id: int, + request: Request, + db: AsyncSession = Depends(get_db), + target_level: str = Form(...), + ai_model: str = Form(...), + generation_count: int = Form(1), + operator_notes: str = Form(""), +): + admin = await _current_admin(request) + if not admin: + return _login_redirect() + + filters = {"status": "", "level": "", "run_id": "", "min_frequency": ""} + basis_item = await db.get(Item, basis_item_id) + if basis_item is None or basis_item.generated_by == "ai" or basis_item.level != "sedang": + return RedirectResponse(url="/admin/basis-items", status_code=HTTP_303_SEE_OTHER) + + if not settings.OPENROUTER_API_KEY: + run_result = await db.execute( + select(AIGenerationRun) + .where(AIGenerationRun.basis_item_id == basis_item.id) + .order_by(AIGenerationRun.id.desc()) + .limit(50) + ) + variant_result = await db.execute( + select(Item) + .where(Item.generated_by == "ai", Item.basis_item_id == basis_item.id) + .order_by(Item.created_at.desc(), Item.id.desc()) + .limit(300) + ) + runs = list(run_result.scalars().all()) + variants = list(variant_result.scalars().all()) + usage_metrics, family_stats = await _family_usage_stats(db, basis_item, variants) + body = _basis_item_workspace_body( + basis_item, + runs, + variants, + usage_metrics, + family_stats, + filters, + error="OPENROUTER_API_KEY is not configured.", + target_level=target_level, + ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, + ) + return _render_admin_page( + f"Basis Item #{basis_item.id}", + f"Basis Item Workspace #{basis_item.id}", + body, + ) + + if target_level not in {"mudah", "sulit"}: + return RedirectResponse(url=f"/admin/basis-items/{basis_item.id}", status_code=HTTP_303_SEE_OTHER) + if not validate_ai_model(ai_model): + return RedirectResponse(url=f"/admin/basis-items/{basis_item.id}", status_code=HTTP_303_SEE_OTHER) + if generation_count < 1 or generation_count > 50: + return RedirectResponse(url=f"/admin/basis-items/{basis_item.id}", status_code=HTTP_303_SEE_OTHER) + + run_id = await create_generation_run( + basis_item_id=basis_item.id, + source_snapshot_question_id=basis_item.source_snapshot_question_id, + target_level=target_level, + requested_count=generation_count, + model=ai_model, + created_by=admin.username, + operator_notes=operator_notes.strip() or None, + db=db, + ) + generated = await generate_questions_batch( + basis_item=basis_item, + target_level=target_level, + ai_model=ai_model, + count=generation_count, + ) + + from app.schemas.ai import GeneratedQuestion + + saved = 0 + for generated_question in generated: + item_id = await save_ai_question( + generated_data=GeneratedQuestion( + stem=generated_question.stem, + options=generated_question.options, + correct=generated_question.correct, + explanation=generated_question.explanation or None, + ), + tryout_id=basis_item.tryout_id, + website_id=basis_item.website_id, + basis_item_id=basis_item.id, + slot=basis_item.slot, + level=target_level, + ai_model=ai_model, + generation_run_id=run_id, + source_snapshot_question_id=basis_item.source_snapshot_question_id, + variant_status="draft", + db=db, + ) + if item_id: + saved += 1 + + await db.commit() + + run_result = await db.execute( + select(AIGenerationRun) + .where(AIGenerationRun.basis_item_id == basis_item.id) + .order_by(AIGenerationRun.id.desc()) + .limit(50) + ) + variant_result = await db.execute( + select(Item) + .where(Item.generated_by == "ai", Item.basis_item_id == basis_item.id) + .order_by(Item.created_at.desc(), Item.id.desc()) + .limit(300) + ) + runs = list(run_result.scalars().all()) + variants = list(variant_result.scalars().all()) + usage_metrics, family_stats = await _family_usage_stats(db, basis_item, variants) + body = _basis_item_workspace_body( + basis_item, + runs, + variants, + usage_metrics, + family_stats, + filters, + success=f"Run #{run_id} finished. Saved {saved} variant(s).", + target_level=target_level, + ai_model=ai_model, + generation_count=str(generation_count), + ) + return _render_admin_page( + f"Basis Item #{basis_item.id}", + f"Basis Item Workspace #{basis_item.id}", + body, + ) + + +@router.post("/basis-items/{basis_item_id}/review-bulk", include_in_schema=False) +async def basis_item_review_bulk( + basis_item_id: int, + request: Request, + db: AsyncSession = Depends(get_db), + item_ids: list[int] = Form([]), + action: str = Form(...), +): + filters = {"status": "", "level": "", "run_id": "", "min_frequency": ""} + admin = await _current_admin(request) + if not admin: + return _login_redirect() + + basis_item = await db.get(Item, basis_item_id) + if basis_item is None: + return RedirectResponse(url="/admin/basis-items", status_code=HTTP_303_SEE_OTHER) + + valid_actions = {"approved", "rejected", "archived", "stale", "active"} + if action in valid_actions and item_ids: + result = await db.execute( + select(Item).where( + Item.id.in_(item_ids), + Item.generated_by == "ai", + Item.basis_item_id == basis_item.id, + ) + ) + items = list(result.scalars().all()) + reviewed_at = datetime.now(timezone.utc) + for item in items: + item.variant_status = action + item.reviewed_by = admin.username + item.reviewed_at = reviewed_at + await db.commit() + + run_result = await db.execute( + select(AIGenerationRun) + .where(AIGenerationRun.basis_item_id == basis_item.id) + .order_by(AIGenerationRun.id.desc()) + .limit(50) + ) + variant_result = await db.execute( + select(Item) + .where(Item.generated_by == "ai", Item.basis_item_id == basis_item.id) + .order_by(Item.created_at.desc(), Item.id.desc()) + .limit(300) + ) + runs = list(run_result.scalars().all()) + variants = list(variant_result.scalars().all()) + usage_metrics, family_stats = await _family_usage_stats(db, basis_item, variants) + body = _basis_item_workspace_body( + basis_item, + runs, + variants, + usage_metrics, + family_stats, + filters, + success=f"Applied status '{action}' to selected variants.", + ) + return _render_admin_page( + f"Basis Item #{basis_item.id}", + f"Basis Item Workspace #{basis_item.id}", + body, + ) + + def _ai_form_body( key_configured: bool, stats: dict[str, Any], error: str | None = None, success: str | None = None, - result: dict[str, Any] | None = None, + generation_summary: dict[str, Any] | None = None, basis_items: list[Item] | None = None, + generation_runs: list[AIGenerationRun] | None = None, + generated_variants: list[Item] | None = None, basis_item_id: str = "", target_level: str = "mudah", ai_model: str = settings.OPENROUTER_MODEL_QWEN, + generation_count: str = "1", + operator_notes: str = "", ) -> str: error_html = f'
{escape(error)}
' if error else "" success_html = f'
{escape(success)}
' if success else "" @@ -1400,6 +1961,8 @@ def _ai_form_body( for model, label in SUPPORTED_MODELS.items() ) basis_items = basis_items or [] + generation_runs = generation_runs or [] + generated_variants = generated_variants or [] basis_rows = [ [ item.id, @@ -1424,48 +1987,73 @@ def _ai_form_body( """ - result_html = "" - if result: - options = result.get("options") or {} - save_html = "" - if result.get("basis_item_id") and not result.get("existing_item_id"): - save_html = f""" -
- - - - - - - - - - - -
- """ - elif result.get("existing_item_id"): - save_html = f""" -
- Slot {escape(str(result.get("slot", "")))} already has a {escape(str(result.get("target_level", "")))} item - for this tryout. Existing item ID: {escape(str(result.get("existing_item_id")))}. -
- """ - result_html = f""" -

Preview Result

-

Model: {escape(result.get("ai_model", ""))}

-

Basis Item: #{escape(str(result.get("basis_item_id", "")))} | Tryout: {escape(result.get("tryout_id", ""))} | Slot: {escape(str(result.get("slot", "")))}

-

Stem:
{escape(result.get("stem", ""))}

-

Options:

- {_table(["Key", "Text"], [[key, value] for key, value in options.items()])} -

Correct: {escape(result.get("correct", ""))}

-

Explanation:
{escape(result.get("explanation", ""))}

-
{save_html}
+ summary_html = "" + if generation_summary: + saved_item_ids = generation_summary.get("saved_item_ids") or [] + summary_html = f""" +

Latest Generation Run

+
+
Run ID{generation_summary.get("run_id", "-")}
+
Requested{generation_summary.get("requested_count", 0)}
+
Generated{generation_summary.get("generated_count", 0)}
+
Saved{len(saved_item_ids)}
+
+

Each saved output starts as draft. Review per item below to approve, reject, archive, stale, or activate.

""" + run_rows = [ + [ + run.id, + run.basis_item_id, + run.target_level, + run.requested_count, + run.model, + run.created_by, + str(run.created_at), + ] + for run in generation_runs + ] + runs_table = _table( + ["Run ID", "Basis Item", "Target", "Requested", "Model", "Created By", "Created At"], + run_rows, + ) + + variant_rows = [] + for item in generated_variants: + variant_rows.append( + "" + f"" + f"{item.id}" + f"{item.generation_run_id or '-'}" + f"{item.basis_item_id or '-'}" + f"{escape(item.level)}" + f"{escape(item.variant_status)}" + f"{escape(item.ai_model or '-')}" + f"{escape(_truncate(item.stem, 100))}" + f"{escape(str(item.created_at))}" + "" + ) + variants_table = ( + "
" + "
" + "" + "" + "
" + "" + + ("".join(variant_rows) if variant_rows else "") + + "
el.checked = this.checked)\">Item IDRun IDBasisLevelStatusModelStemCreated At
No AI-generated variants yet.
" + ) + return f"""

OpenRouter key configured: {"Yes" if key_configured else "No"}

Total AI-generated items: {stats.get("total_ai_items", 0)}

+

Hybrid workflow: one run can generate one or many variants; each item remains independently reviewable.

{success_html} {error_html} {seed_callout} @@ -1481,12 +2069,22 @@ def _ai_form_body( - + + +

Recommended: 1-3 variants per run. Larger runs can increase overlap and review burden. Backend safety cap: 50.

+ + +

Available Sedang Basis Items

The generator needs a sedang item. Use one of these IDs, or seed demo data if the table is empty.

{basis_table} - {result_html} + {summary_html} +

Recent Generation Runs

+ {runs_table} +

Generated Variants (Review Queue)

+

Use bulk review actions to move items from draft to approved/active, or reject/archive/stale.

+ {variants_table} """ @@ -1499,10 +2097,20 @@ async def ai_playground_view(request: Request, db: AsyncSession = Depends(get_db stats = await get_ai_stats(db) basis_items = await _basis_items_for_playground(db) basis_item_id = request.query_params.get("basis_item_id", "") + generation_runs = await _recent_generation_runs(db) + selected_basis_item_id: int | None = None + if basis_item_id and str(basis_item_id).isdigit(): + selected_basis_item_id = int(str(basis_item_id)) + generated_variants = await _recent_generated_variants( + db, + basis_item_id=selected_basis_item_id, + ) body = _ai_form_body( bool(settings.OPENROUTER_API_KEY), stats, basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id or ""), ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1517,11 +2125,15 @@ async def ai_playground_seed_demo(request: Request, db: AsyncSession = Depends(g demo_item = await _find_or_create_demo_basis_item(db) stats = await get_ai_stats(db) basis_items = await _basis_items_for_playground(db) + generation_runs = await _recent_generation_runs(db) + generated_variants = await _recent_generated_variants(db) body = _ai_form_body( bool(settings.OPENROUTER_API_KEY), stats, success=f"Demo basis item is ready: item #{demo_item.id}, tryout {demo_item.tryout_id}, slot {demo_item.slot}.", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(demo_item.id), ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1534,6 +2146,8 @@ async def ai_playground_submit( basis_item_id: int = Form(...), target_level: str = Form(...), ai_model: str = Form(...), + generation_count: int = Form(1), + operator_notes: str = Form(""), ): admin = await _current_admin(request) if not admin: @@ -1541,6 +2155,8 @@ async def ai_playground_submit( stats = await get_ai_stats(db) basis_items = await _basis_items_for_playground(db) + generation_runs = await _recent_generation_runs(db) + generated_variants = await _recent_generated_variants(db) if not settings.OPENROUTER_API_KEY: body = _ai_form_body( @@ -1548,9 +2164,13 @@ async def ai_playground_submit( stats, error="OPENROUTER_API_KEY is not configured in the environment.", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1560,9 +2180,13 @@ async def ai_playground_submit( stats, error="Target level must be mudah or sulit.", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1572,9 +2196,13 @@ async def ai_playground_submit( stats, error="Unsupported AI model.", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1586,9 +2214,13 @@ async def ai_playground_submit( stats, error=f"Basis item not found: {basis_item_id}", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1598,59 +2230,115 @@ async def ai_playground_submit( stats, error=f"Basis item must be sedang level, got: {basis_item.level}", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) - generated = await generate_question( - basis_item=basis_item, - target_level=target_level, - ai_model=ai_model, - ) - if not generated: + if generation_count < 1 or generation_count > 50: body = _ai_form_body( True, stats, - error="AI generation failed. Check OPENROUTER_API_KEY, model availability, and server logs.", + error="Generate count must be between 1 and 50.", basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) - existing_item_result = await db.execute( - select(Item.id).where( - Item.tryout_id == basis_item.tryout_id, - Item.website_id == basis_item.website_id, - Item.slot == basis_item.slot, - Item.level == target_level, - ) + run_id = await create_generation_run( + basis_item_id=basis_item.id, + source_snapshot_question_id=basis_item.source_snapshot_question_id, + target_level=target_level, + requested_count=generation_count, + model=ai_model, + created_by=admin.username, + operator_notes=operator_notes.strip() or None, + db=db, ) - existing_item_id = existing_item_result.scalar_one_or_none() + + generated = await generate_questions_batch( + basis_item=basis_item, + target_level=target_level, + ai_model=ai_model, + count=generation_count, + ) + + saved_item_ids: list[int] = [] + from app.schemas.ai import GeneratedQuestion + + for generated_question in generated: + item_id = await save_ai_question( + generated_data=GeneratedQuestion( + stem=generated_question.stem, + options=generated_question.options, + correct=generated_question.correct, + explanation=generated_question.explanation or None, + ), + tryout_id=basis_item.tryout_id, + website_id=basis_item.website_id, + basis_item_id=basis_item.id, + slot=basis_item.slot, + level=target_level, + ai_model=ai_model, + generation_run_id=run_id, + source_snapshot_question_id=basis_item.source_snapshot_question_id, + variant_status="draft", + db=db, + ) + if item_id: + saved_item_ids.append(item_id) + + await db.commit() + updated_stats = await get_ai_stats(db) + updated_basis_items = await _basis_items_for_playground(db) + updated_runs = await _recent_generation_runs(db) + updated_variants = await _recent_generated_variants(db) + + if not saved_item_ids: + body = _ai_form_body( + True, + updated_stats, + error="Generation run completed but no items were saved. Check model output and logs.", + basis_items=updated_basis_items, + generation_runs=updated_runs, + generated_variants=updated_variants, + basis_item_id=str(basis_item_id), + target_level=target_level, + ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, + ) + return _render_admin_page("AI Playground", "AI Playground", body) body = _ai_form_body( True, - stats, - basis_items=basis_items, - result={ - "basis_item_id": basis_item.id, - "tryout_id": basis_item.tryout_id, - "website_id": basis_item.website_id, - "slot": basis_item.slot, - "target_level": target_level, - "stem": generated.stem, - "options": generated.options, - "correct": generated.correct, - "explanation": generated.explanation or "", - "ai_model": ai_model, - "existing_item_id": existing_item_id, + updated_stats, + success=f"Generation run #{run_id} completed. Saved {len(saved_item_ids)} item(s).", + generation_summary={ + "run_id": run_id, + "requested_count": generation_count, + "generated_count": len(generated), + "saved_item_ids": saved_item_ids, }, + basis_items=updated_basis_items, + generation_runs=updated_runs, + generated_variants=updated_variants, basis_item_id=str(basis_item_id), target_level=target_level, ai_model=ai_model, + generation_count=str(generation_count), + operator_notes=operator_notes, ) return _render_admin_page("AI Playground", "AI Playground", body) @@ -1697,24 +2385,6 @@ async def ai_playground_save( ) return _render_admin_page("AI Playground", "AI Playground", body) - existing_result = await db.execute( - select(Item.id).where( - Item.tryout_id == tryout_id, - Item.website_id == website_id, - Item.slot == slot, - Item.level == target_level, - ) - ) - existing_item_id = existing_result.scalar_one_or_none() - if existing_item_id: - body = _ai_form_body( - bool(settings.OPENROUTER_API_KEY), - stats, - error=f"Item already exists at tryout={tryout_id}, slot={slot}, level={target_level} (item #{existing_item_id}).", - basis_items=basis_items, - ) - return _render_admin_page("AI Playground", "AI Playground", body) - from app.schemas.ai import GeneratedQuestion item_id = await save_ai_question( @@ -1730,6 +2400,7 @@ async def ai_playground_save( slot=slot, level=target_level, ai_model=ai_model, + variant_status="draft", db=db, ) if not item_id: @@ -1756,6 +2427,83 @@ async def ai_playground_save( return _render_admin_page("AI Playground", "AI Playground", body) +@router.post("/ai-playground/review-bulk", include_in_schema=False) +async def ai_playground_review_bulk( + request: Request, + db: AsyncSession = Depends(get_db), + item_ids: list[int] = Form([]), + action: str = Form(...), +): + admin = await _current_admin(request) + if not admin: + return _login_redirect() + + valid_actions = {"approved", "rejected", "archived", "stale", "active"} + stats = await get_ai_stats(db) + basis_items = await _basis_items_for_playground(db) + generation_runs = await _recent_generation_runs(db) + generated_variants = await _recent_generated_variants(db) + + if action not in valid_actions: + body = _ai_form_body( + bool(settings.OPENROUTER_API_KEY), + stats, + error="Invalid review action.", + basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, + ) + return _render_admin_page("AI Playground", "AI Playground", body) + + if not item_ids: + body = _ai_form_body( + bool(settings.OPENROUTER_API_KEY), + stats, + error="Select at least one generated item.", + basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, + ) + return _render_admin_page("AI Playground", "AI Playground", body) + + result = await db.execute( + select(Item).where(Item.id.in_(item_ids), Item.generated_by == "ai") + ) + items = list(result.scalars().all()) + if not items: + body = _ai_form_body( + bool(settings.OPENROUTER_API_KEY), + stats, + error="No matching AI-generated items were found for the selected IDs.", + basis_items=basis_items, + generation_runs=generation_runs, + generated_variants=generated_variants, + ) + return _render_admin_page("AI Playground", "AI Playground", body) + + reviewed_at = datetime.now(timezone.utc) + for item in items: + item.variant_status = action + item.reviewed_by = admin.username + item.reviewed_at = reviewed_at + + await db.commit() + + updated_stats = await get_ai_stats(db) + updated_basis_items = await _basis_items_for_playground(db) + updated_runs = await _recent_generation_runs(db) + updated_variants = await _recent_generated_variants(db) + body = _ai_form_body( + bool(settings.OPENROUTER_API_KEY), + updated_stats, + success=f"Updated {len(items)} item(s) to status '{action}'.", + basis_items=updated_basis_items, + generation_runs=updated_runs, + generated_variants=updated_variants, + ) + return _render_admin_page("AI Playground", "AI Playground", body) + + @router.get("/tryout/list", include_in_schema=False) @router.get("/item/list", include_in_schema=False) @router.get("/user/list", include_in_schema=False) diff --git a/app/models/__init__.py b/app/models/__init__.py index 660425e..9563f75 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -5,6 +5,7 @@ Exports all SQLAlchemy ORM models for use in the application. """ from app.database import Base +from app.models.ai_generation_run import AIGenerationRun from app.models.item import Item from app.models.session import Session from app.models.tryout import Tryout @@ -17,6 +18,7 @@ from app.models.website import Website __all__ = [ "Base", + "AIGenerationRun", "User", "Website", "Tryout", diff --git a/app/models/ai_generation_run.py b/app/models/ai_generation_run.py new file mode 100644 index 0000000..8d5b81f --- /dev/null +++ b/app/models/ai_generation_run.py @@ -0,0 +1,72 @@ +""" +AI generation run model. + +Represents one admin generation request that can produce one or many variants. +""" + +from datetime import datetime +from typing import Optional + +from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.database import Base + + +class AIGenerationRun(Base): + __tablename__ = "ai_generation_runs" + + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + basis_item_id: Mapped[int] = mapped_column( + ForeignKey("items.id", ondelete="CASCADE", onupdate="CASCADE"), + nullable=False, + index=True, + comment="Basis item ID", + ) + source_snapshot_question_id: Mapped[Optional[int]] = mapped_column( + ForeignKey("tryout_snapshot_questions.id", ondelete="SET NULL", onupdate="CASCADE"), + nullable=True, + index=True, + comment="Source snapshot question ID", + ) + target_level: Mapped[str] = mapped_column( + String(50), + nullable=False, + comment="Target level (mudah/sulit)", + ) + requested_count: Mapped[int] = mapped_column( + Integer, + nullable=False, + default=1, + comment="Requested output count", + ) + model: Mapped[str] = mapped_column( + String(255), + nullable=False, + comment="Model identifier", + ) + prompt_version: Mapped[str] = mapped_column( + String(50), + nullable=False, + default="v1", + comment="Prompt template version", + ) + operator_notes: Mapped[Optional[str]] = mapped_column( + Text, + nullable=True, + comment="Optional admin notes", + ) + created_by: Mapped[str] = mapped_column( + String(255), + nullable=False, + comment="Admin username", + ) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + generated_items: Mapped[list["Item"]] = relationship( + "Item", + back_populates="generation_run", + lazy="selectin", + ) diff --git a/app/models/item.py b/app/models/item.py index 1d35a6b..f9b5b4c 100644 --- a/app/models/item.py +++ b/app/models/item.py @@ -155,6 +155,39 @@ class Item(Base): nullable=True, comment="Original item ID (for AI variants)", ) + generation_run_id: Mapped[Union[int, None]] = mapped_column( + ForeignKey("ai_generation_runs.id", ondelete="SET NULL", onupdate="CASCADE"), + nullable=True, + index=True, + comment="AI generation run ID", + ) + source_snapshot_question_id: Mapped[Union[int, None]] = mapped_column( + ForeignKey("tryout_snapshot_questions.id", ondelete="SET NULL", onupdate="CASCADE"), + nullable=True, + index=True, + comment="Source snapshot question ID", + ) + variant_status: Mapped[str] = mapped_column( + String(50), + nullable=False, + default="active", + comment="Lifecycle status (active/draft/approved/rejected/archived/stale)", + ) + reviewed_by: Mapped[Union[str, None]] = mapped_column( + String(255), + nullable=True, + comment="Reviewer username", + ) + reviewed_at: Mapped[Union[datetime, None]] = mapped_column( + DateTime(timezone=True), + nullable=True, + comment="Review timestamp", + ) + review_notes: Mapped[Union[str, None]] = mapped_column( + Text, + nullable=True, + comment="Review notes", + ) # Timestamps created_at: Mapped[datetime] = mapped_column( @@ -187,6 +220,11 @@ class Item(Base): lazy="selectin", cascade="all, delete-orphan", ) + generation_run: Mapped[Union["AIGenerationRun", None]] = relationship( + "AIGenerationRun", + back_populates="generated_items", + lazy="selectin", + ) # Constraints and indexes __table_args__ = ( @@ -203,10 +241,11 @@ class Item(Base): "website_id", "slot", "level", - unique=True, + unique=False, ), Index("ix_items_calibrated", "calibrated"), Index("ix_items_basis_item_id", "basis_item_id"), + Index("ix_items_variant_status", "variant_status"), # IRT b parameter constraint [-3, +3] CheckConstraint( "irt_b IS NULL OR (irt_b >= -3 AND irt_b <= 3)", diff --git a/app/services/ai_generation.py b/app/services/ai_generation.py index 8993f89..1c2cf5c 100644 --- a/app/services/ai_generation.py +++ b/app/services/ai_generation.py @@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import get_settings from app.models.item import Item +from app.models.ai_generation_run import AIGenerationRun from app.models.tryout import Tryout from app.models.user_answer import UserAnswer from app.schemas.ai import GeneratedQuestion @@ -493,6 +494,9 @@ async def save_ai_question( level: Literal["mudah", "sedang", "sulit"], ai_model: str, db: AsyncSession, + generation_run_id: int | None = None, + source_snapshot_question_id: int | None = None, + variant_status: str = "draft", ) -> Optional[int]: """ Save AI-generated question to database. @@ -523,6 +527,9 @@ async def save_ai_question( generated_by="ai", ai_model=ai_model, basis_item_id=basis_item_id, + generation_run_id=generation_run_id, + source_snapshot_question_id=source_snapshot_question_id, + variant_status=variant_status, calibrated=False, ctt_p=None, ctt_bobot=None, @@ -547,6 +554,50 @@ async def save_ai_question( return None +async def create_generation_run( + basis_item_id: int, + target_level: Literal["mudah", "sulit"], + requested_count: int, + model: str, + created_by: str, + db: AsyncSession, + source_snapshot_question_id: int | None = None, + operator_notes: str | None = None, + prompt_version: str = "v1", +) -> int: + run = AIGenerationRun( + basis_item_id=basis_item_id, + source_snapshot_question_id=source_snapshot_question_id, + target_level=target_level, + requested_count=requested_count, + model=model, + prompt_version=prompt_version, + operator_notes=operator_notes, + created_by=created_by, + ) + db.add(run) + await db.flush() + return int(run.id) + + +async def generate_questions_batch( + basis_item: Item, + target_level: Literal["mudah", "sulit"], + ai_model: str, + count: int, +) -> list[GeneratedQuestion]: + generated_items: list[GeneratedQuestion] = [] + for _ in range(count): + generated = await generate_question( + basis_item=basis_item, + target_level=target_level, + ai_model=ai_model, + ) + if generated is not None: + generated_items.append(generated) + return generated_items + + async def get_ai_stats(db: AsyncSession) -> Dict[str, Any]: """ Get AI generation statistics. diff --git a/app/services/tryout_json_import.py b/app/services/tryout_json_import.py index 7751841..6c2fb38 100644 --- a/app/services/tryout_json_import.py +++ b/app/services/tryout_json_import.py @@ -17,7 +17,7 @@ from typing import Any from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.models import TryoutImportSnapshot, TryoutSnapshotQuestion, Website +from app.models import Item, TryoutImportSnapshot, TryoutSnapshotQuestion, Website SOURCE_FORMAT = "sejoli_json" DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" @@ -292,7 +292,8 @@ async def import_tryout_json_snapshot(payload: dict[str, Any], website_id: int, new_questions += 1 continue - if existing.content_checksum != question["content_checksum"]: + content_changed = existing.content_checksum != question["content_checksum"] + if content_changed: existing.question_title = question["title"] or question["question"] existing.question_html = question["question"] existing.explanation_html = question["explanation"] @@ -313,6 +314,18 @@ async def import_tryout_json_snapshot(payload: dict[str, Any], website_id: int, existing.is_active = True existing.last_seen_at = now + # If source content changed, mark AI children derived from this source as stale. + if content_changed: + stale_variants_result = await db.execute( + select(Item).where( + Item.generated_by == "ai", + Item.source_snapshot_question_id == existing.id, + Item.variant_status.in_(["draft", "approved", "active"]), + ) + ) + for variant in stale_variants_result.scalars().all(): + variant.variant_status = "stale" + removed_questions = 0 for source_question_id, existing in existing_questions.items(): if existing.is_active and source_question_id not in incoming_ids: @@ -320,6 +333,15 @@ async def import_tryout_json_snapshot(payload: dict[str, Any], website_id: int, existing.latest_snapshot_id = snapshot.id existing.last_seen_at = now removed_questions += 1 + stale_removed_result = await db.execute( + select(Item).where( + Item.generated_by == "ai", + Item.source_snapshot_question_id == existing.id, + Item.variant_status.in_(["draft", "approved", "active"]), + ) + ) + for variant in stale_removed_result.scalars().all(): + variant.variant_status = "stale" imported_tryouts.append( {