Checkpoint React frontend migration
This commit is contained in:
Dwindi Ramadhana
424
backend/app/routers/import_export.py
Normal file
424
backend/app/routers/import_export.py
Normal file
@@ -0,0 +1,424 @@
|
||||
"""
|
||||
Import/Export API router for migration and snapshot ingestion.
|
||||
|
||||
Endpoints:
|
||||
- POST /api/v1/import/preview: Preview Excel import without saving
|
||||
- POST /api/v1/import/questions: Import questions from Excel to database
|
||||
- GET /api/v1/export/questions: Export questions to Excel file
|
||||
- POST /api/v1/import-export/tryout-json/preview: Preview Sejoli tryout JSON import
|
||||
- POST /api/v1/import-export/tryout-json: Import Sejoli tryout JSON as read-only snapshot
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import json
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile, status
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.auth import AuthContext, get_auth_context, require_website_auth
|
||||
from app.core.rate_limit import enforce_rate_limit
|
||||
from app.database import get_db
|
||||
from app.models import Website
|
||||
from app.services.excel_import import (
|
||||
bulk_insert_items,
|
||||
export_questions_to_excel,
|
||||
parse_excel_import,
|
||||
validate_excel_structure,
|
||||
)
|
||||
from app.services.tryout_json_import import (
|
||||
TryoutImportError,
|
||||
import_tryout_json_snapshot,
|
||||
preview_tryout_json_import,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/import-export", tags=["import-export"])
|
||||
|
||||
|
||||
async def ensure_website_exists(
|
||||
website_id: int,
|
||||
db: AsyncSession,
|
||||
) -> None:
|
||||
website = await db.get(Website, website_id)
|
||||
if website is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=(
|
||||
f"Website {website_id} not found. Website registration is stored in the database, "
|
||||
"not in .env."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/preview",
|
||||
summary="Preview Excel import",
|
||||
description="Parse Excel file and return preview without saving to database.",
|
||||
)
|
||||
async def preview_import(
|
||||
request: Request,
|
||||
file: UploadFile = File(..., description="Excel file (.xlsx)"),
|
||||
auth: AuthContext = Depends(get_auth_context),
|
||||
) -> dict:
|
||||
"""
|
||||
Preview Excel import without saving to database.
|
||||
|
||||
Args:
|
||||
file: Excel file upload (.xlsx format)
|
||||
website_id: Website ID from header
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- items_count: Number of items parsed
|
||||
- preview: List of item previews
|
||||
- validation_errors: List of validation errors if any
|
||||
|
||||
Raises:
|
||||
HTTPException: If file format is invalid or parsing fails
|
||||
"""
|
||||
website_id = require_website_auth(auth, allowed_roles={"admin", "system_admin"})
|
||||
await enforce_rate_limit(
|
||||
request,
|
||||
scope="import.preview",
|
||||
max_requests=30,
|
||||
window_seconds=300,
|
||||
)
|
||||
|
||||
# Validate file format
|
||||
if not file.filename or not file.filename.lower().endswith('.xlsx'):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="File must be .xlsx format",
|
||||
)
|
||||
|
||||
# Save uploaded file to temporary location
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".xlsx") as temp_file:
|
||||
content = await file.read()
|
||||
temp_file.write(content)
|
||||
temp_file_path = temp_file.name
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save uploaded file: {str(e)}",
|
||||
)
|
||||
|
||||
try:
|
||||
# Validate Excel structure
|
||||
validation = validate_excel_structure(temp_file_path)
|
||||
if not validation["valid"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={
|
||||
"error": "Invalid Excel structure",
|
||||
"validation_errors": validation["errors"],
|
||||
},
|
||||
)
|
||||
|
||||
# Parse Excel (tryout_id is optional for preview)
|
||||
tryout_id = "preview" # Use dummy tryout_id for preview
|
||||
result = parse_excel_import(
|
||||
temp_file_path,
|
||||
website_id=website_id,
|
||||
tryout_id=tryout_id
|
||||
)
|
||||
|
||||
if result["validation_errors"]:
|
||||
return {
|
||||
"items_count": result["items_count"],
|
||||
"preview": result["items"],
|
||||
"validation_errors": result["validation_errors"],
|
||||
"has_errors": True,
|
||||
}
|
||||
|
||||
# Return limited preview (first 5 items)
|
||||
preview_items = result["items"][:5]
|
||||
|
||||
return {
|
||||
"items_count": result["items_count"],
|
||||
"preview": preview_items,
|
||||
"validation_errors": [],
|
||||
"has_errors": False,
|
||||
}
|
||||
|
||||
finally:
|
||||
# Clean up temporary file
|
||||
if os.path.exists(temp_file_path):
|
||||
os.unlink(temp_file_path)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/questions",
|
||||
summary="Import questions from Excel",
|
||||
description="Parse Excel file and import questions to database with 100% data integrity.",
|
||||
)
|
||||
async def import_questions(
|
||||
request: Request,
|
||||
file: UploadFile = File(..., description="Excel file (.xlsx)"),
|
||||
auth: AuthContext = Depends(get_auth_context),
|
||||
tryout_id: str = Form(..., description="Tryout identifier"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""
|
||||
Import questions from Excel to database.
|
||||
|
||||
Validates file format, parses Excel content, checks for duplicates,
|
||||
and performs bulk insert with rollback on error.
|
||||
|
||||
Args:
|
||||
file: Excel file upload (.xlsx format)
|
||||
website_id: Website ID from header
|
||||
tryout_id: Tryout identifier
|
||||
db: Async database session
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- imported: Number of items successfully imported
|
||||
- duplicates: Number of duplicate items skipped
|
||||
- errors: List of errors if any
|
||||
|
||||
Raises:
|
||||
HTTPException: If file format is invalid, validation fails, or import fails
|
||||
"""
|
||||
website_id = require_website_auth(auth, allowed_roles={"admin", "system_admin"})
|
||||
await enforce_rate_limit(
|
||||
request,
|
||||
scope="import.questions",
|
||||
max_requests=20,
|
||||
window_seconds=300,
|
||||
)
|
||||
|
||||
# Validate file format
|
||||
if not file.filename or not file.filename.lower().endswith('.xlsx'):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="File must be .xlsx format",
|
||||
)
|
||||
|
||||
# Save uploaded file to temporary location
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".xlsx") as temp_file:
|
||||
content = await file.read()
|
||||
temp_file.write(content)
|
||||
temp_file_path = temp_file.name
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Failed to save uploaded file: {str(e)}",
|
||||
)
|
||||
|
||||
try:
|
||||
# Validate Excel structure
|
||||
validation = validate_excel_structure(temp_file_path)
|
||||
if not validation["valid"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={
|
||||
"error": "Invalid Excel structure",
|
||||
"validation_errors": validation["errors"],
|
||||
},
|
||||
)
|
||||
|
||||
# Parse Excel
|
||||
result = parse_excel_import(
|
||||
temp_file_path,
|
||||
website_id=website_id,
|
||||
tryout_id=tryout_id
|
||||
)
|
||||
|
||||
# Check for validation errors
|
||||
if result["validation_errors"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail={
|
||||
"error": "Validation failed",
|
||||
"validation_errors": result["validation_errors"],
|
||||
},
|
||||
)
|
||||
|
||||
# Check if items were parsed
|
||||
if result["items_count"] == 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="No items found in Excel file",
|
||||
)
|
||||
|
||||
# Bulk insert items
|
||||
insert_result = await bulk_insert_items(result["items"], db)
|
||||
|
||||
# Check for insertion errors
|
||||
if insert_result["errors"]:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail={
|
||||
"error": "Import failed",
|
||||
"errors": insert_result["errors"],
|
||||
},
|
||||
)
|
||||
|
||||
# Check for conflicts (duplicates)
|
||||
if insert_result["duplicate_count"] > 0:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail={
|
||||
"message": f"Import completed with {insert_result['duplicate_count']} duplicate(s) skipped",
|
||||
"imported": insert_result["inserted_count"],
|
||||
"duplicates": insert_result["duplicate_count"],
|
||||
},
|
||||
)
|
||||
|
||||
return {
|
||||
"message": "Import successful",
|
||||
"imported": insert_result["inserted_count"],
|
||||
"duplicates": insert_result["duplicate_count"],
|
||||
}
|
||||
|
||||
finally:
|
||||
# Clean up temporary file
|
||||
if os.path.exists(temp_file_path):
|
||||
os.unlink(temp_file_path)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/export/questions",
|
||||
summary="Export questions to Excel",
|
||||
description="Export questions for a tryout to Excel file in standardized format.",
|
||||
)
|
||||
async def export_questions(
|
||||
tryout_id: str,
|
||||
auth: AuthContext = Depends(get_auth_context),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> FileResponse:
|
||||
"""
|
||||
Export questions to Excel file.
|
||||
|
||||
Args:
|
||||
tryout_id: Tryout identifier
|
||||
website_id: Website ID from header
|
||||
db: Async database session
|
||||
|
||||
Returns:
|
||||
FileResponse with Excel file
|
||||
|
||||
Raises:
|
||||
HTTPException: If tryout has no questions or export fails
|
||||
"""
|
||||
website_id = require_website_auth(auth, allowed_roles={"admin", "system_admin"})
|
||||
|
||||
try:
|
||||
# Export questions to Excel
|
||||
output_path = await export_questions_to_excel(
|
||||
tryout_id=tryout_id,
|
||||
website_id=website_id,
|
||||
db=db
|
||||
)
|
||||
|
||||
# Return file for download
|
||||
filename = f"tryout_{tryout_id}_questions.xlsx"
|
||||
return FileResponse(
|
||||
path=output_path,
|
||||
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
filename=filename,
|
||||
)
|
||||
|
||||
except ValueError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(e),
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Export failed: {str(e)}",
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/tryout-json/preview",
|
||||
summary="Preview Sejoli tryout JSON import",
|
||||
description="Parse a Sejoli tryout export JSON file and show snapshot diff without writing to database.",
|
||||
)
|
||||
async def preview_tryout_json(
|
||||
request: Request,
|
||||
file: UploadFile = File(..., description="Sejoli tryout export JSON"),
|
||||
auth: AuthContext = Depends(get_auth_context),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
website_id = require_website_auth(auth, allowed_roles={"admin", "system_admin"})
|
||||
await enforce_rate_limit(
|
||||
request,
|
||||
scope="import.tryout_json_preview",
|
||||
max_requests=30,
|
||||
window_seconds=300,
|
||||
)
|
||||
if not file.filename or not file.filename.lower().endswith(".json"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="File must be .json format",
|
||||
)
|
||||
|
||||
await ensure_website_exists(website_id, db)
|
||||
|
||||
try:
|
||||
payload = json.loads((await file.read()).decode("utf-8"))
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON file: {str(e)}",
|
||||
)
|
||||
|
||||
try:
|
||||
return await preview_tryout_json_import(payload, website_id, db)
|
||||
except TryoutImportError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(e),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/tryout-json",
|
||||
summary="Import Sejoli tryout JSON snapshot",
|
||||
description="Store Sejoli tryout export JSON as read-only snapshot data and upsert normalized reference questions.",
|
||||
)
|
||||
async def import_tryout_json(
|
||||
request: Request,
|
||||
file: UploadFile = File(..., description="Sejoli tryout export JSON"),
|
||||
auth: AuthContext = Depends(get_auth_context),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
website_id = require_website_auth(auth, allowed_roles={"admin", "system_admin"})
|
||||
if website_id is None:
|
||||
x_website_id = request.headers.get("x-website-id")
|
||||
if not x_website_id or not x_website_id.isdigit():
|
||||
raise HTTPException(status_code=400, detail="X-Website-ID header is required for system_admin")
|
||||
website_id = int(x_website_id)
|
||||
await enforce_rate_limit(
|
||||
request,
|
||||
scope="import.tryout_json",
|
||||
max_requests=20,
|
||||
window_seconds=300,
|
||||
)
|
||||
if not file.filename or not file.filename.lower().endswith(".json"):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="File must be .json format",
|
||||
)
|
||||
|
||||
await ensure_website_exists(website_id, db)
|
||||
|
||||
try:
|
||||
payload = json.loads((await file.read()).decode("utf-8"))
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=f"Invalid JSON file: {str(e)}",
|
||||
)
|
||||
|
||||
try:
|
||||
return await import_tryout_json_snapshot(payload, website_id, db)
|
||||
except TryoutImportError as e:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(e),
|
||||
)
|
||||
Reference in New Issue
Block a user