feat: 全量功能模块开发与集成测试修复
- 新增后端模块：Alert、APIAsset、Compliance、Lineage、Masking、Risk、SchemaChange、Unstructured、Watermark - 新增前端模块页面与API接口 - 新增Alembic迁移脚本(002-014)覆盖全量业务表 - 新增测试数据生成脚本与集成测试脚本 - 修复metadata模型JSON类型导入缺失导致启动失败的问题 - 修复前端Alert/APIAsset页面request模块路径错误 - 更新docker-compose与开发计划文档
This commit is contained in:
hiderfong
@@ -2,6 +2,14 @@ from app.models.user import User, Role, Dept, UserRole
|
||||
from app.models.metadata import DataSource, Database, DataTable, DataColumn, UnstructuredFile
|
||||
from app.models.classification import Category, DataLevel, RecognitionRule, ClassificationTemplate
|
||||
from app.models.project import ClassificationProject, ClassificationTask, ClassificationResult, ClassificationChange
|
||||
from app.models.ml import MLModelVersion
|
||||
from app.models.masking import MaskingRule
|
||||
from app.models.watermark import WatermarkLog
|
||||
from app.models.schema_change import SchemaChangeLog
|
||||
from app.models.risk import RiskAssessment
|
||||
from app.models.compliance import ComplianceRule, ComplianceIssue
|
||||
from app.models.alert import AlertRule, AlertRecord, WorkOrder
|
||||
from app.models.api_asset import APIAsset, APIEndpoint
|
||||
from app.models.log import OperationLog
|
||||
|
||||
__all__ = [
|
||||
@@ -9,5 +17,12 @@ __all__ = [
|
||||
"DataSource", "Database", "DataTable", "DataColumn", "UnstructuredFile",
|
||||
"Category", "DataLevel", "RecognitionRule", "ClassificationTemplate",
|
||||
"ClassificationProject", "ClassificationTask", "ClassificationResult", "ClassificationChange",
|
||||
"MLModelVersion",
|
||||
"MaskingRule",
|
||||
"WatermarkLog",
|
||||
"SchemaChangeLog",
|
||||
"RiskAssessment",
|
||||
"ComplianceRule",
|
||||
"ComplianceIssue",
|
||||
"OperationLog",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, ForeignKey, JSON
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class AlertRule(Base):
|
||||
__tablename__ = "alert_rule"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(200), nullable=False)
|
||||
trigger_condition = Column(String(50), nullable=False) # l5_count, risk_score, schema_change
|
||||
threshold = Column(Integer, default=0)
|
||||
severity = Column(String(20), default="medium")
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class AlertRecord(Base):
|
||||
__tablename__ = "alert_record"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
rule_id = Column(Integer, ForeignKey("alert_rule.id"), nullable=False)
|
||||
title = Column(String(200), nullable=False)
|
||||
content = Column(Text)
|
||||
severity = Column(String(20), default="medium")
|
||||
status = Column(String(20), default="open") # open, acknowledged, resolved
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
rule = relationship("AlertRule")
|
||||
|
||||
|
||||
class WorkOrder(Base):
|
||||
__tablename__ = "work_order"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
alert_id = Column(Integer, ForeignKey("alert_record.id"), nullable=True)
|
||||
title = Column(String(200), nullable=False)
|
||||
description = Column(Text)
|
||||
assignee_id = Column(Integer, ForeignKey("sys_user.id"), nullable=True)
|
||||
status = Column(String(20), default="open") # open, in_progress, resolved
|
||||
resolution = Column(Text)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
resolved_at = Column(DateTime, nullable=True)
|
||||
|
||||
assignee = relationship("User")
|
||||
@@ -0,0 +1,41 @@
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, JSON, ForeignKey, BigInteger
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.core.database import Base
|
||||
from datetime import datetime
|
||||
|
||||
class APIAsset(Base):
|
||||
__tablename__ = "api_asset"
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(200), nullable=False)
|
||||
base_url = Column(String(500), nullable=False)
|
||||
swagger_url = Column(String(500), nullable=True)
|
||||
auth_type = Column(String(50), default="none") # none, bearer, api_key, basic
|
||||
headers = Column(JSON, default=dict)
|
||||
description = Column(Text, nullable=True)
|
||||
scan_status = Column(String(20), default="idle") # idle, scanning, completed, failed
|
||||
total_endpoints = Column(Integer, default=0)
|
||||
sensitive_endpoints = Column(Integer, default=0)
|
||||
created_by = Column(Integer, ForeignKey("sys_user.id"), nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
endpoints = relationship("APIEndpoint", back_populates="asset", cascade="all, delete-orphan")
|
||||
creator = relationship("User", foreign_keys=[created_by])
|
||||
|
||||
class APIEndpoint(Base):
|
||||
__tablename__ = "api_endpoint"
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
asset_id = Column(Integer, ForeignKey("api_asset.id"), nullable=False)
|
||||
method = Column(String(10), nullable=False) # GET, POST, PUT, DELETE, etc.
|
||||
path = Column(String(500), nullable=False)
|
||||
summary = Column(String(500), nullable=True)
|
||||
tags = Column(JSON, default=list)
|
||||
parameters = Column(JSON, default=list)
|
||||
request_body_schema = Column(JSON, nullable=True)
|
||||
response_schema = Column(JSON, nullable=True)
|
||||
sensitive_fields = Column(JSON, default=list) # detected PII fields
|
||||
risk_level = Column(String(20), default="low") # low, medium, high, critical
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
asset = relationship("APIAsset", back_populates="endpoints")
|
||||
@@ -0,0 +1,33 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, JSON
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class ComplianceRule(Base):
|
||||
__tablename__ = "compliance_rule"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(200), nullable=False)
|
||||
standard = Column(String(50), nullable=False) # dengbao, pipl, gdpr
|
||||
description = Column(Text)
|
||||
check_logic = Column(String(50), nullable=False) # check_masking, check_encryption, check_audit, check_level
|
||||
severity = Column(String(20), default="medium") # low, medium, high, critical
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class ComplianceIssue(Base):
|
||||
__tablename__ = "compliance_issue"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
rule_id = Column(Integer, nullable=False)
|
||||
project_id = Column(Integer, nullable=True)
|
||||
entity_type = Column(String(20), nullable=False) # project, source, column
|
||||
entity_id = Column(Integer, nullable=False)
|
||||
entity_name = Column(String(200))
|
||||
severity = Column(String(20), default="medium")
|
||||
description = Column(Text)
|
||||
suggestion = Column(Text)
|
||||
status = Column(String(20), default="open") # open, resolved, ignored
|
||||
resolved_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
@@ -0,0 +1,16 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class DataLineage(Base):
|
||||
__tablename__ = "data_lineage"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
source_table = Column(String(200), nullable=False)
|
||||
source_column = Column(String(200), nullable=True)
|
||||
target_table = Column(String(200), nullable=False)
|
||||
target_column = Column(String(200), nullable=True)
|
||||
relation_type = Column(String(20), default="direct") # direct, derived, lookup
|
||||
script_content = Column(Text)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
@@ -0,0 +1,22 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, JSON, Text
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class MaskingRule(Base):
|
||||
__tablename__ = "masking_rule"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(100), nullable=False)
|
||||
level_id = Column(Integer, ForeignKey("data_level.id"), nullable=True)
|
||||
category_id = Column(Integer, ForeignKey("category.id"), nullable=True)
|
||||
algorithm = Column(String(20), nullable=False) # mask, truncate, hash, generalize, replace
|
||||
params = Column(JSON, default=dict) # algorithm-specific params
|
||||
is_active = Column(Boolean, default=True)
|
||||
description = Column(Text)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
level = relationship("DataLevel")
|
||||
category = relationship("Category")
|
||||
@@ -1,5 +1,5 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Text, BigInteger
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Text, BigInteger, JSON
|
||||
from sqlalchemy.orm import relationship
|
||||
|
||||
from app.core.database import Base
|
||||
@@ -36,6 +36,10 @@ class Database(Base):
|
||||
charset = Column(String(50))
|
||||
table_count = Column(Integer, default=0)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
last_scanned_at = Column(DateTime, nullable=True)
|
||||
checksum = Column(String(64), nullable=True)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
deleted_at = Column(DateTime, nullable=True)
|
||||
|
||||
source = relationship("DataSource", back_populates="databases")
|
||||
tables = relationship("DataTable", back_populates="database", cascade="all, delete-orphan")
|
||||
@@ -51,6 +55,10 @@ class DataTable(Base):
|
||||
row_count = Column(BigInteger, default=0)
|
||||
column_count = Column(Integer, default=0)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
last_scanned_at = Column(DateTime, nullable=True)
|
||||
checksum = Column(String(64), nullable=True)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
deleted_at = Column(DateTime, nullable=True)
|
||||
|
||||
database = relationship("Database", back_populates="tables")
|
||||
columns = relationship("DataColumn", back_populates="table", cascade="all, delete-orphan")
|
||||
@@ -68,6 +76,10 @@ class DataColumn(Base):
|
||||
is_nullable = Column(Boolean, default=True)
|
||||
sample_data = Column(Text) # JSON array of sample values
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
last_scanned_at = Column(DateTime, nullable=True)
|
||||
checksum = Column(String(64), nullable=True)
|
||||
is_deleted = Column(Boolean, default=False)
|
||||
deleted_at = Column(DateTime, nullable=True)
|
||||
|
||||
table = relationship("DataTable", back_populates="columns")
|
||||
|
||||
@@ -81,6 +93,7 @@ class UnstructuredFile(Base):
|
||||
file_size = Column(BigInteger)
|
||||
storage_path = Column(String(500))
|
||||
extracted_text = Column(Text)
|
||||
analysis_result = Column(JSON, nullable=True) # JSON: {matches: [{rule_name, category, level, snippet}]}
|
||||
status = Column(String(20), default="pending") # pending, processed, error
|
||||
created_by = Column(Integer, ForeignKey("sys_user.id"))
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, Text
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class MLModelVersion(Base):
|
||||
__tablename__ = "ml_model_version"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(100), nullable=False)
|
||||
model_path = Column(String(500), nullable=False) # joblib dump path
|
||||
vectorizer_path = Column(String(500), nullable=False) # tfidf vectorizer path
|
||||
accuracy = Column(Float, default=0.0)
|
||||
train_samples = Column(Integer, default=0)
|
||||
train_date = Column(DateTime, default=datetime.utcnow)
|
||||
is_active = Column(Boolean, default=False)
|
||||
description = Column(Text)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
@@ -48,6 +48,10 @@ class ClassificationProject(Base):
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
# Async classification tracking
|
||||
celery_task_id = Column(String(100), nullable=True)
|
||||
scan_progress = Column(Text, nullable=True) # JSON: {"scanned": 0, "matched": 0, "total": 0}
|
||||
|
||||
template = relationship("ClassificationTemplate")
|
||||
tasks = relationship("ClassificationTask", back_populates="project", cascade="all, delete-orphan")
|
||||
results = relationship("ClassificationResult", back_populates="project", cascade="all, delete-orphan")
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Float, DateTime, ForeignKey, JSON
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class RiskAssessment(Base):
|
||||
__tablename__ = "risk_assessment"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
entity_type = Column(String(20), nullable=False) # project, source, table, field
|
||||
entity_id = Column(Integer, nullable=False)
|
||||
entity_name = Column(String(200))
|
||||
risk_score = Column(Float, default=0.0) # 0-100
|
||||
sensitivity_score = Column(Float, default=0.0)
|
||||
exposure_score = Column(Float, default=0.0)
|
||||
protection_score = Column(Float, default=0.0)
|
||||
details = Column(JSON, default=dict)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
@@ -0,0 +1,23 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class SchemaChangeLog(Base):
|
||||
__tablename__ = "schema_change_log"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
source_id = Column(Integer, ForeignKey("data_source.id"), nullable=False)
|
||||
database_id = Column(Integer, ForeignKey("meta_database.id"), nullable=True)
|
||||
table_id = Column(Integer, ForeignKey("meta_table.id"), nullable=True)
|
||||
column_id = Column(Integer, ForeignKey("meta_column.id"), nullable=True)
|
||||
change_type = Column(String(20), nullable=False) # add_table, drop_table, add_column, drop_column, change_type, change_comment
|
||||
old_value = Column(Text)
|
||||
new_value = Column(Text)
|
||||
detected_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
source = relationship("DataSource")
|
||||
database = relationship("Database")
|
||||
table = relationship("DataTable")
|
||||
column = relationship("DataColumn")
|
||||
@@ -0,0 +1,17 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class WatermarkLog(Base):
|
||||
__tablename__ = "watermark_log"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(Integer, ForeignKey("sys_user.id"), nullable=False)
|
||||
export_type = Column(String(20), default="csv") # csv, excel, txt
|
||||
data_scope = Column(Text) # JSON: {source_id, table_name, row_count}
|
||||
watermark_key = Column(String(64), nullable=False) # random key for this export
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
user = relationship("User")
|
||||
Reference in New Issue
Block a user