hiderfong/prop-data-guard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 全量功能模块开发与集成测试修复

Browse Source 
- 新增后端模块：Alert、APIAsset、Compliance、Lineage、Masking、Risk、SchemaChange、Unstructured、Watermark
- 新增前端模块页面与API接口
- 新增Alembic迁移脚本(002-014)覆盖全量业务表
- 新增测试数据生成脚本与集成测试脚本
- 修复metadata模型JSON类型导入缺失导致启动失败的问题
- 修复前端Alert/APIAsset页面request模块路径错误
- 更新docker-compose与开发计划文档
This commit is contained in:
hiderfong
2026-04-25 08:51:38 +08:00
parent 8b2bc84399
commit 6d70520e79
110 changed files with 6125 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/alembic/versions/002_fix_encryption.py
+41
View File
@@ -0,0 +1,41 @@
"""Fix datasource password encryption stability
Revision ID: 002
Revises: 001
Create Date: 2026-04-23 14:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "002"
down_revision: Union[str, None] = "001"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Historical encrypted_password values are irrecoverable because
# the old implementation generated a random Fernet key on every startup.
# We clear the passwords and mark sources as inactive so admins re-enter them
# with the new stable key derived from DB_ENCRYPTION_KEY / SECRET_KEY.
op.add_column(
"data_source",
sa.Column("password_reset_required", sa.Boolean(), nullable=False, server_default=sa.text("false")),
)
op.execute(
"""
UPDATE data_source
SET encrypted_password = NULL,
status = 'inactive',
password_reset_required = true
WHERE encrypted_password IS NOT NULL
"""
)
def downgrade() -> None:
op.drop_column("data_source", "password_reset_required")
backend/alembic/versions/003_add_project_task_fields.py
+27
View File
@@ -0,0 +1,27 @@
"""Add celery_task_id and scan_progress to classification_project
Revision ID: 003
Revises: 002
Create Date: 2026-04-23 14:30:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "003"
down_revision: Union[str, None] = "002"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("classification_project", sa.Column("celery_task_id", sa.String(100), nullable=True))
op.add_column("classification_project", sa.Column("scan_progress", sa.Text(), nullable=True))
def downgrade() -> None:
op.drop_column("classification_project", "scan_progress")
op.drop_column("classification_project", "celery_task_id")
backend/alembic/versions/004_add_ml_model.py
+37
View File
@@ -0,0 +1,37 @@
"""Add ml_model_version table
Revision ID: 004
Revises: 003
Create Date: 2026-04-23 15:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "004"
down_revision: Union[str, None] = "003"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"ml_model_version",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("name", sa.String(100), nullable=False),
sa.Column("model_path", sa.String(500), nullable=False),
sa.Column("vectorizer_path", sa.String(500), nullable=False),
sa.Column("accuracy", sa.Float(), default=0.0),
sa.Column("train_samples", sa.Integer(), default=0),
sa.Column("train_date", sa.DateTime(), default=sa.func.now()),
sa.Column("is_active", sa.Boolean(), default=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("ml_model_version")
backend/alembic/versions/005_add_incremental_meta.py
+33
View File
@@ -0,0 +1,33 @@
"""Add incremental scan fields to meta tables
Revision ID: 005
Revises: 004
Create Date: 2026-04-23 16:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "005"
down_revision: Union[str, None] = "004"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
for table in ["meta_database", "meta_table", "meta_column"]:
op.add_column(table, sa.Column("last_scanned_at", sa.DateTime(), nullable=True))
op.add_column(table, sa.Column("checksum", sa.String(64), nullable=True))
op.add_column(table, sa.Column("is_deleted", sa.Boolean(), nullable=False, server_default=sa.text("false")))
op.add_column(table, sa.Column("deleted_at", sa.DateTime(), nullable=True))
def downgrade() -> None:
for table in ["meta_database", "meta_table", "meta_column"]:
op.drop_column(table, "deleted_at")
op.drop_column(table, "is_deleted")
op.drop_column(table, "checksum")
op.drop_column(table, "last_scanned_at")
backend/alembic/versions/006_add_masking_rule.py
+37
View File
@@ -0,0 +1,37 @@
"""Add masking_rule table
Revision ID: 006
Revises: 005
Create Date: 2026-04-23 17:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "006"
down_revision: Union[str, None] = "005"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"masking_rule",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("name", sa.String(100), nullable=False),
sa.Column("level_id", sa.Integer(), sa.ForeignKey("data_level.id"), nullable=True),
sa.Column("category_id", sa.Integer(), sa.ForeignKey("category.id"), nullable=True),
sa.Column("algorithm", sa.String(20), nullable=False),
sa.Column("params", sa.JSON(), nullable=True),
sa.Column("is_active", sa.Boolean(), default=True),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(), default=sa.func.now(), onupdate=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("masking_rule")
backend/alembic/versions/007_add_watermark_log.py
+33
View File
@@ -0,0 +1,33 @@
"""Add watermark_log table
Revision ID: 007
Revises: 006
Create Date: 2026-04-23 18:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "007"
down_revision: Union[str, None] = "006"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"watermark_log",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("user_id", sa.Integer(), sa.ForeignKey("sys_user.id"), nullable=False),
sa.Column("export_type", sa.String(20), default="csv"),
sa.Column("data_scope", sa.Text(), nullable=True),
sa.Column("watermark_key", sa.String(64), nullable=False),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("watermark_log")
backend/alembic/versions/008_add_unstructured_analysis.py
+25
View File
@@ -0,0 +1,25 @@
"""Add analysis_result to unstructured_file
Revision ID: 008
Revises: 007
Create Date: 2026-04-23 19:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "008"
down_revision: Union[str, None] = "007"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column("unstructured_file", sa.Column("analysis_result", sa.JSON(), nullable=True))
def downgrade() -> None:
op.drop_column("unstructured_file", "analysis_result")
backend/alembic/versions/009_add_schema_change_log.py
+36
View File
@@ -0,0 +1,36 @@
"""Add schema_change_log table
Revision ID: 009
Revises: 008
Create Date: 2026-04-23 20:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "009"
down_revision: Union[str, None] = "008"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"schema_change_log",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("source_id", sa.Integer(), sa.ForeignKey("data_source.id"), nullable=False),
sa.Column("database_id", sa.Integer(), sa.ForeignKey("meta_database.id"), nullable=True),
sa.Column("table_id", sa.Integer(), sa.ForeignKey("meta_table.id"), nullable=True),
sa.Column("column_id", sa.Integer(), sa.ForeignKey("meta_column.id"), nullable=True),
sa.Column("change_type", sa.String(20), nullable=False),
sa.Column("old_value", sa.Text(), nullable=True),
sa.Column("new_value", sa.Text(), nullable=True),
sa.Column("detected_at", sa.DateTime(), default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("schema_change_log")
backend/alembic/versions/010_add_risk_assessment.py
+40
View File
@@ -0,0 +1,40 @@
"""Add risk_assessment table
Revision ID: 010
Revises: 009
Create Date: 2026-04-23 21:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "010"
down_revision: Union[str, None] = "009"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"risk_assessment",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("entity_type", sa.String(20), nullable=False),
sa.Column("entity_id", sa.Integer(), nullable=False),
sa.Column("entity_name", sa.String(200), nullable=True),
sa.Column("risk_score", sa.Float(), default=0.0),
sa.Column("sensitivity_score", sa.Float(), default=0.0),
sa.Column("exposure_score", sa.Float(), default=0.0),
sa.Column("protection_score", sa.Float(), default=0.0),
sa.Column("details", sa.JSON(), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(), default=sa.func.now(), onupdate=sa.func.now()),
)
op.create_index("idx_risk_entity", "risk_assessment", ["entity_type", "entity_id"])
def downgrade() -> None:
op.drop_index("idx_risk_entity", table_name="risk_assessment")
op.drop_table("risk_assessment")
backend/alembic/versions/011_add_compliance.py
+51
View File
@@ -0,0 +1,51 @@
"""Add compliance_rule and compliance_issue tables
Revision ID: 011
Revises: 010
Create Date: 2026-04-23 22:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "011"
down_revision: Union[str, None] = "010"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"compliance_rule",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("standard", sa.String(50), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("check_logic", sa.String(50), nullable=False),
sa.Column("severity", sa.String(20), default="medium"),
sa.Column("is_active", sa.Boolean(), default=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
op.create_table(
"compliance_issue",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("rule_id", sa.Integer(), nullable=False),
sa.Column("project_id", sa.Integer(), nullable=True),
sa.Column("entity_type", sa.String(20), nullable=False),
sa.Column("entity_id", sa.Integer(), nullable=False),
sa.Column("entity_name", sa.String(200), nullable=True),
sa.Column("severity", sa.String(20), default="medium"),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("suggestion", sa.Text(), nullable=True),
sa.Column("status", sa.String(20), default="open"),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("compliance_issue")
op.drop_table("compliance_rule")
backend/alembic/versions/012_add_data_lineage.py
+35
View File
@@ -0,0 +1,35 @@
"""Add data_lineage table
Revision ID: 012
Revises: 011
Create Date: 2026-04-23 23:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "012"
down_revision: Union[str, None] = "011"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"data_lineage",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("source_table", sa.String(200), nullable=False),
sa.Column("source_column", sa.String(200), nullable=True),
sa.Column("target_table", sa.String(200), nullable=False),
sa.Column("target_column", sa.String(200), nullable=True),
sa.Column("relation_type", sa.String(20), default="direct"),
sa.Column("script_content", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("data_lineage")
backend/alembic/versions/013_add_alert_work_order.py
+58
View File
@@ -0,0 +1,58 @@
"""Add alert and work_order tables
Revision ID: 013
Revises: 012
Create Date: 2026-04-24 00:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "013"
down_revision: Union[str, None] = "012"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"alert_rule",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("trigger_condition", sa.String(50), nullable=False),
sa.Column("threshold", sa.Integer(), default=0),
sa.Column("severity", sa.String(20), default="medium"),
sa.Column("is_active", sa.Boolean(), default=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
op.create_table(
"alert_record",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("rule_id", sa.Integer(), sa.ForeignKey("alert_rule.id"), nullable=False),
sa.Column("title", sa.String(200), nullable=False),
sa.Column("content", sa.Text(), nullable=True),
sa.Column("severity", sa.String(20), default="medium"),
sa.Column("status", sa.String(20), default="open"),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
op.create_table(
"work_order",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("alert_id", sa.Integer(), sa.ForeignKey("alert_record.id"), nullable=True),
sa.Column("title", sa.String(200), nullable=False),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("assignee_id", sa.Integer(), sa.ForeignKey("sys_user.id"), nullable=True),
sa.Column("status", sa.String(20), default="open"),
sa.Column("resolution", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
sa.Column("resolved_at", sa.DateTime(), nullable=True),
)
def downgrade() -> None:
op.drop_table("work_order")
op.drop_table("alert_record")
op.drop_table("alert_rule")
backend/alembic/versions/014_add_api_asset.py
+55
View File
@@ -0,0 +1,55 @@
"""Add api_asset and api_endpoint tables
Revision ID: 014
Revises: 013
Create Date: 2026-04-24 00:00:00.000000
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "014"
down_revision: Union[str, None] = "013"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
"api_asset",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("name", sa.String(200), nullable=False),
sa.Column("base_url", sa.String(500), nullable=False),
sa.Column("swagger_url", sa.String(500), nullable=True),
sa.Column("auth_type", sa.String(50), default="none"),
sa.Column("headers", sa.JSON(), default=dict),
sa.Column("description", sa.Text(), nullable=True),
sa.Column("scan_status", sa.String(20), default="idle"),
sa.Column("total_endpoints", sa.Integer(), default=0),
sa.Column("sensitive_endpoints", sa.Integer(), default=0),
sa.Column("created_by", sa.Integer(), sa.ForeignKey("sys_user.id"), nullable=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
sa.Column("updated_at", sa.DateTime(), default=sa.func.now(), onupdate=sa.func.now()),
)
op.create_table(
"api_endpoint",
sa.Column("id", sa.Integer(), primary_key=True, index=True),
sa.Column("asset_id", sa.Integer(), sa.ForeignKey("api_asset.id"), nullable=False),
sa.Column("method", sa.String(10), nullable=False),
sa.Column("path", sa.String(500), nullable=False),
sa.Column("summary", sa.String(500), nullable=True),
sa.Column("tags", sa.JSON(), default=list),
sa.Column("parameters", sa.JSON(), default=list),
sa.Column("request_body_schema", sa.JSON(), nullable=True),
sa.Column("response_schema", sa.JSON(), nullable=True),
sa.Column("sensitive_fields", sa.JSON(), default=list),
sa.Column("risk_level", sa.String(20), default="low"),
sa.Column("is_active", sa.Boolean(), default=True),
sa.Column("created_at", sa.DateTime(), default=sa.func.now()),
)
def downgrade() -> None:
op.drop_table("api_endpoint")
op.drop_table("api_asset")