fix: optimize compliance scan performance and improve error handling

- Refactor scan_compliance to eliminate N+1 queries using joinedload and batch loading - Add try-except wrapper in compliance scan API endpoint - Improve frontend axios error interceptor to display detail/message/timeout errors - Update CORS config and nginx for domain deployment
2026-04-25 20:49:38 +08:00
parent 6d35cfa5b7
commit 34466a1ae9
10 changed files with 690 additions and 105 deletions
@@ -1,5 +1,5 @@
 from typing import Optional
-from fastapi import APIRouter, Depends, Query
+from fastapi import APIRouter, Depends, Query, HTTPException, status
 from sqlalchemy.orm import Session

 from app.core.database import get_db
@@ -26,8 +26,16 @@ def scan_compliance(
    db: Session = Depends(get_db),
    current_user: User = Depends(get_current_user),
 ):
-    issues = compliance_service.scan_compliance(db, project_id=project_id)
-    return ResponseModel(data={"issues_found": len(issues)})
+    try:
+        issues = compliance_service.scan_compliance(db, project_id=project_id)
+        return ResponseModel(data={"issues_found": len(issues)})
+    except Exception:
+        import logging
+        logging.exception("Compliance scan failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail="扫描执行失败，请稍后重试"
+        )


@router.get("/issues")
@@ -67,6 +75,5 @@ def resolve_issue(
 ):
    issue = compliance_service.resolve_issue(db, issue_id)
    if not issue:
-        from fastapi import HTTPException, status
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="问题不存在")
    return ResponseModel(message="已标记为已解决")
@@ -1,5 +1,5 @@
 from typing import Optional
-from fastapi import APIRouter, Depends, Query
+from fastapi import APIRouter, Depends, Query, HTTPException, status
 from sqlalchemy.orm import Session

 from app.core.database import get_db
@@ -89,7 +89,6 @@ def delete_project(
 ):
    p = project_service.get_project(db, project_id)
    if not p:
-        from fastapi import HTTPException, status
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="项目不存在")
    # Only admin or project creator can delete
    if not _is_admin(current_user) and p.created_by != current_user.id:
@@ -110,32 +109,39 @@ def project_auto_classify(

    project = project_service.get_project(db, project_id)
    if not project:
-        from fastapi import HTTPException, status
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="项目不存在")

-    if background:
-        # Check if already running
-        if project.celery_task_id:
-            existing = AsyncResult(project.celery_task_id)
-            if existing.state in ("PENDING", "PROGRESS", "STARTED"):
-                return ResponseModel(data={"task_id": project.celery_task_id, "status": existing.state})
+    try:
+        if background:
+            # Check if already running
+            if project.celery_task_id:
+                existing = AsyncResult(project.celery_task_id)
+                if existing.state in ("PENDING", "PROGRESS", "STARTED"):
+                    return ResponseModel(data={"task_id": project.celery_task_id, "status": existing.state})

-        task = auto_classify_task.delay(project_id)
-        project.celery_task_id = task.id
-        project.status = "scanning"
-        db.commit()
-        return ResponseModel(data={"task_id": task.id, "status": task.state})
-    else:
-        from app.services.classification_engine import run_auto_classification
-        project.status = "scanning"
-        db.commit()
-        result = run_auto_classification(db, project_id)
-        if result.get("success"):
-            project.status = "assigning"
+            task = auto_classify_task.delay(project_id)
+            project.celery_task_id = task.id
+            project.status = "scanning"
+            db.commit()
+            return ResponseModel(data={"task_id": task.id, "status": task.state})
        else:
-            project.status = "created"
-        db.commit()
-        return ResponseModel(data=result)
+            from app.services.classification_engine import run_auto_classification
+            project.status = "scanning"
+            db.commit()
+            result = run_auto_classification(db, project_id)
+            if result.get("success"):
+                project.status = "assigning"
+            else:
+                project.status = "created"
+            db.commit()
+            return ResponseModel(data=result)
+    except Exception as e:
+        import logging
+        logging.exception("Auto classify failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"自动分类执行失败: {str(e)}"
+        )


@router.get("/{project_id}/auto-classify-status")
@@ -149,7 +155,6 @@ def project_auto_classify_status(

    project = project_service.get_project(db, project_id)
    if not project:
-        from fastapi import HTTPException, status
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="项目不存在")

    task_id = project.celery_task_id
@@ -39,19 +39,9 @@ async def log_requests(request: Request, call_next):
        return response

    from app.core.database import SessionLocal
+    db = None
    try:
        db = SessionLocal()
-        body_bytes = b""
-        if request.method in ["POST", "PUT", "PATCH"]:
-            try:
-                body_bytes = await request.body()
-                # Re-assign body for downstream
-                async def receive():
-                    return {"type": "http.request", "body": body_bytes}
-                request._receive = receive
-            except Exception:
-                pass
-
        log_entry = log_models.OperationLog(
            module=request.url.path.split("/")[2] if len(request.url.path.split("/")) > 2 else "",
            action=request.url.path,
@@ -66,7 +56,8 @@ async def log_requests(request: Request, call_next):
    except Exception:
        pass
    finally:
-        db.close()
+        if db:
+            db.close()

    return response

@@ -1,10 +1,9 @@
-from typing import List, Optional
-from sqlalchemy.orm import Session
 from datetime import datetime
+from typing import List, Optional, Set, Tuple
+from sqlalchemy.orm import Session, joinedload

 from app.models.compliance import ComplianceRule, ComplianceIssue
-from app.models.project import ClassificationProject, ClassificationResult
-from app.models.classification import DataLevel
+from app.models.project import ClassificationResult
 from app.models.masking import MaskingRule


@@ -26,79 +25,98 @@ def init_builtin_rules(db: Session):
 def scan_compliance(db: Session, project_id: Optional[int] = None) -> List[ComplianceIssue]:
    """Run compliance scan and generate issues."""
    rules = db.query(ComplianceRule).filter(ComplianceRule.is_active == True).all()
-    issues = []
+    if not rules:
+        return []

    # Get masking rules for check_masking logic
    masking_rules = db.query(MaskingRule).filter(MaskingRule.is_active == True).all()
    masking_level_ids = {r.level_id for r in masking_rules if r.level_id}

-    query = db.query(ClassificationProject)
+    # Build result filter and determine project ids
+    result_filter = [ClassificationResult.level_id.isnot(None)]
+    project_ids: List[int] = []
    if project_id:
-        query = query.filter(ClassificationProject.id == project_id)
-    projects = query.all()
+        result_filter.append(ClassificationResult.project_id == project_id)
+        project_ids = [project_id]
+    else:
+        project_ids = [
+            r[0] for r in db.query(ClassificationResult.project_id).distinct().all()
+        ]
+        if project_ids:
+            result_filter.append(ClassificationResult.project_id.in_(project_ids))
+        else:
+            return []

-    for project in projects:
-        results = db.query(ClassificationResult).filter(
-            ClassificationResult.project_id == project.id,
-            ClassificationResult.level_id.isnot(None),
-        ).all()
+    # Pre-load all results with level and column to avoid N+1 queries
+    results = db.query(ClassificationResult).options(
+        joinedload(ClassificationResult.level),
+        joinedload(ClassificationResult.column),
+    ).filter(*result_filter).all()

-        for r in results:
-            if not r.level:
-                continue
-            level_code = r.level.code
+    if not results:
+        return []

-            for rule in rules:
-                matched = False
-                desc = ""
-                suggestion = ""
+    # Batch query existing open issues
+    existing_issues = db.query(ComplianceIssue).filter(
+        ComplianceIssue.project_id.in_(project_ids),
+        ComplianceIssue.status == "open",
+    ).all()
+    existing_set: Set[Tuple[int, int, str, int]] = {
+        (i.rule_id, i.project_id, i.entity_type, i.entity_id) for i in existing_issues
+    }

-                if rule.check_logic == "check_masking" and level_code in ("L4", "L5"):
-                    if r.level_id not in masking_level_ids:
-                        matched = True
-                        desc = f"字段 '{r.column.name if r.column else '未知'}' 为 {level_code} 级，但未配置脱敏规则"
-                        suggestion = "请在【数据脱敏】模块为该分级配置脱敏策略"
+    issues = []
+    for r in results:
+        if not r.level:
+            continue
+        level_code = r.level.code

-                elif rule.check_logic == "check_encryption" and level_code == "L5":
-                    # Placeholder: no encryption check in MVP, always flag
+        for rule in rules:
+            matched = False
+            desc = ""
+            suggestion = ""
+
+            if rule.check_logic == "check_masking" and level_code in ("L4", "L5"):
+                if r.level_id not in masking_level_ids:
                    matched = True
-                    desc = f"字段 '{r.column.name if r.column else '未知'}' 为 L5 级核心数据，建议确认是否加密存储"
-                    suggestion = "请确认该字段在数据库中已加密存储"
+                    desc = f"字段 '{r.column.name if r.column else '未知'}' 为 {level_code} 级，但未配置脱敏规则"
+                    suggestion = "请在【数据脱敏】模块为该分级配置脱敏策略"

-                elif rule.check_logic == "check_level" and level_code in ("L4", "L5"):
-                    if r.source == "auto":
-                        matched = True
-                        desc = f"个人敏感字段 '{r.column.name if r.column else '未知'}' 目前为自动识别，建议人工复核并确认授权"
-                        suggestion = "请人工确认该字段的处理已取得合法授权"
+            elif rule.check_logic == "check_encryption" and level_code == "L5":
+                # Placeholder: no encryption check in MVP, always flag
+                matched = True
+                desc = f"字段 '{r.column.name if r.column else '未知'}' 为 L5 级核心数据，建议确认是否加密存储"
+                suggestion = "请确认该字段在数据库中已加密存储"

-                elif rule.check_logic == "check_audit":
-                    # Placeholder for cross-border check
-                    pass
+            elif rule.check_logic == "check_level" and level_code in ("L4", "L5"):
+                if r.source == "auto":
+                    matched = True
+                    desc = f"个人敏感字段 '{r.column.name if r.column else '未知'}' 目前为自动识别，建议人工复核并确认授权"
+                    suggestion = "请人工确认该字段的处理已取得合法授权"

-                if matched:
-                    # Check if open issue already exists
-                    existing = db.query(ComplianceIssue).filter(
-                        ComplianceIssue.rule_id == rule.id,
-                        ComplianceIssue.project_id == project.id,
-                        ComplianceIssue.entity_type == "column",
-                        ComplianceIssue.entity_id == (r.column_id or 0),
-                        ComplianceIssue.status == "open",
-                    ).first()
-                    if not existing:
-                        issue = ComplianceIssue(
-                            rule_id=rule.id,
-                            project_id=project.id,
-                            entity_type="column",
-                            entity_id=r.column_id or 0,
-                            entity_name=r.column.name if r.column else "未知",
-                            severity=rule.severity,
-                            description=desc,
-                            suggestion=suggestion,
-                        )
-                        db.add(issue)
-                        issues.append(issue)
+            elif rule.check_logic == "check_audit":
+                # Placeholder for cross-border check
+                pass

-    db.commit()
+            if matched:
+                key = (rule.id, r.project_id, "column", r.column_id or 0)
+                if key not in existing_set:
+                    issue = ComplianceIssue(
+                        rule_id=rule.id,
+                        project_id=r.project_id,
+                        entity_type="column",
+                        entity_id=r.column_id or 0,
+                        entity_name=r.column.name if r.column else "未知",
+                        severity=rule.severity,
+                        description=desc,
+                        suggestion=suggestion,
+                    )
+                    db.add(issue)
+                    issues.append(issue)
+                    existing_set.add(key)
+
+    if issues:
+        db.commit()
    return issues