From 16cca50e2cf77827a9af3a571123ecc481907be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A2=81=E6=B7=8B=E6=B6=9B?= Date: Mon, 16 Nov 2026 11:06:48 +0800 Subject: [PATCH 1/3] =?UTF-8?q?refactor(db):=20=E5=B0=86=20SQL=20=E5=8F=AA?= =?UTF-8?q?=E8=AF=BB=E6=A0=A1=E9=AA=8C=E6=8B=86=E5=88=86=E4=B8=BA=E6=96=B9?= =?UTF-8?q?=E8=A8=80=E6=B3=A8=E5=86=8C=E8=A1=A8=E4=B8=8E=E5=8F=AF=E6=89=A9?= =?UTF-8?q?=E5=B1=95=E8=A7=84=E5=88=99=E6=B5=81=E6=B0=B4=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 apps/db/dialect:按 DB 类型注册 DialectPolicy(sqlglot 方言、危险函数、资源类函数、超时、是否允许改写 LIMIT),未注册类型一律拒绝 - 新增 apps/db/sql_guard:SqlCheck / SqlRewriter / GuardPipeline;basic 规则集保持 check_sql_read 原有语义,agent 规则集增加单语句、嵌套写操作、锁、函数黑名单、表白名单、列权限检查,以及确定性行权限改写和行数上限 - check_sql_read、get_sqlglot_dialect、get_dangerous_functions 保留签名并改为委托注册表;excel、kingbase、redshift 现与 pg 共用文件读写类危险函数黑名单 - exec_sql 新增可选 timeout_seconds、max_rows,不传时行为不变;py_driver 执行分支合并为按类型登记的执行配置 AI-Coding: true AI-Tool: TophantAI --- backend/apps/db/db.py | 403 ++++++++++--------------- backend/apps/db/dialect/__init__.py | 20 ++ backend/apps/db/dialect/base.py | 48 +++ backend/apps/db/dialect/builtin.py | 88 ++++++ backend/apps/db/dialect/registry.py | 41 +++ backend/apps/db/sql_guard/__init__.py | 26 ++ backend/apps/db/sql_guard/base.py | 158 ++++++++++ backend/apps/db/sql_guard/basic.py | 120 ++++++++ backend/apps/db/sql_guard/profiles.py | 56 ++++ backend/apps/db/sql_guard/rewriters.py | 77 +++++ backend/apps/db/sql_guard/strict.py | 209 +++++++++++++ backend/apps/db/sql_guard/tables.py | 68 +++++ backend/tests/test_sql_guard.py | 278 +++++++++++++++++ 13 files changed, 1355 insertions(+), 237 deletions(-) create mode 100644 backend/apps/db/dialect/__init__.py create mode 100644 backend/apps/db/dialect/base.py create mode 100644 backend/apps/db/dialect/builtin.py create mode 100644 backend/apps/db/dialect/registry.py create mode 100644 backend/apps/db/sql_guard/__init__.py create mode 100644 backend/apps/db/sql_guard/base.py create mode 100644 backend/apps/db/sql_guard/basic.py create mode 100644 backend/apps/db/sql_guard/profiles.py create mode 100644 backend/apps/db/sql_guard/rewriters.py create mode 100644 backend/apps/db/sql_guard/strict.py create mode 100644 backend/apps/db/sql_guard/tables.py create mode 100644 backend/tests/test_sql_guard.py diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index e37d5695..20b52e63 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -24,6 +24,10 @@ from apps.datasource.models.datasource import DatasourceConf, CoreDatasource, TableSchema, ColumnSchema from apps.datasource.utils.utils import aes_decrypt from apps.db.constant import DB, ConnectType +from apps.db.dialect import dialect_registry +from apps.db.sql_guard import BASIC as GUARD_BASIC, SqlRejected, build_context as build_guard_context, \ + get_pipeline as get_guard_pipeline +from apps.db.sql_guard import basic as sql_guard_basic from apps.db.engine import get_engine_config from apps.system.crud.assistant import get_out_ds_conf from apps.system.schemas.system_schema import AssistantOutDsSchema @@ -877,7 +881,31 @@ def convert_mssql_value(value, col_idx, string_col_idx_set, is_mssql): return value -def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column=False): +def _fetch_rows(cursor_or_result, max_rows: Optional[int]): + """fetchall(),或在给定上限时最多取 max_rows 行(用于不能改写 LIMIT 的方言)。""" + if max_rows is None: + return cursor_or_result.fetchall() + return cursor_or_result.fetchmany(max_rows) + + +def _apply_statement_timeout(session, ds, timeout_seconds: Optional[int]): + """按方言策略设置单条语句超时;返回恢复回调。方言不支持时返回 None 且不报错。""" + if not timeout_seconds: + return None + policy = dialect_registry.find(ds.type) + if policy is None or policy.apply_timeout is None: + SQLBotLogUtil.warning(f"Statement timeout is not supported for datasource type {ds.type}") + return None + return policy.apply_timeout(session.connection(), int(timeout_seconds)) + + +def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column=False, + timeout_seconds: Optional[int] = None, max_rows: Optional[int] = None): + """执行只读 SQL。 + + timeout_seconds:按方言策略设置单条语句超时(仅 sqlalchemy 连接、且方言支持时生效)。 + max_rows:最多读取的行数,用于无法通过改写 SQL 限行的方言;不传则读取全部(原有行为)。 + """ while sql.endswith(';'): sql = sql[:-1] # check execute sql only contain read operations @@ -890,150 +918,120 @@ def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column= with get_session(ds) as session: # 获取当前数据库方言 dialect_name = session.bind.dialect.name + restore_timeout = _apply_statement_timeout(session, ds, timeout_seconds) + try: + return _exec_sqlalchemy(session, sql, dialect_name, origin_column, max_rows) + finally: + if restore_timeout is not None: + try: + restore_timeout() + except Exception: + SQLBotLogUtil.exception("Failed to restore statement timeout") + return _exec_driver(ds, sql, origin_column, max_rows) - with session.execute(text(sql)) as result: - try: - columns = result.keys()._keys if origin_column else [item.lower() for item in result.keys()._keys] - # SQL Server varchar 等字符串列需做 GBK 编码转换 - is_mssql = dialect_name == 'mssql' - string_cols = [] +def _encode_sql(sql: str) -> str: + return bytes.decode(base64.b64encode(bytes(sql, 'utf-8'))) - fields_info = [] - for col_idx, col_name in enumerate(columns): - is_numeric = False - try: - type_code = result.cursor.description[col_idx][1] - is_numeric = is_numeric_type_code(type_code, dialect_name) - if is_mssql and is_mssql_string_type_code(type_code): - string_cols.append(col_idx) - except (IndexError, AttributeError): - pass +def _exec_sqlalchemy(session, sql: str, dialect_name: str, origin_column, max_rows: Optional[int]): + with session.execute(text(sql)) as result: + try: + columns = result.keys()._keys if origin_column else [item.lower() for item in result.keys()._keys] - fields_info.append({ - "name": col_name, - "is_numeric": is_numeric - }) + # SQL Server varchar 等字符串列需做 GBK 编码转换 + is_mssql = dialect_name == 'mssql' + string_cols = [] - string_col_idx_set = set(string_cols) + fields_info = [] - res = result.fetchall() - result_list = [ - {str(columns[i]): convert_mssql_value(convert_value(value), i, string_col_idx_set, is_mssql) - for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} - except Exception as ex: - raise ParseSQLResultError(str(ex)) - else: - conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) - # extra_config_dict = get_extra_config(conf) - if equals_ignore_case(ds.type, 'dm'): - with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: - try: - cursor.execute(sql, timeout=conf.timeout) - res = cursor.fetchall() - columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for - field in - cursor.description] - fields_info = build_fields_info_from_cursor(cursor, origin_column, 'dm') - result_list = [ - {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} - except Exception as ex: - raise ParseSQLResultError(str(ex)) - elif equals_ignore_case(ds.type, 'doris', 'starrocks'): - with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: - try: - cursor.execute(sql) - res = cursor.fetchall() - columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for - field in - cursor.description] - fields_info = build_fields_info_from_cursor(cursor, origin_column, 'mysql') - result_list = [ - {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} - except Exception as ex: - raise ParseSQLResultError(str(ex)) - elif equals_ignore_case(ds.type, 'redshift'): - with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: - try: - cursor.execute(sql) - res = cursor.fetchall() - columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for - field in - cursor.description] - fields_info = build_fields_info_from_cursor(cursor, origin_column, 'postgresql') - result_list = [ - {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} - except Exception as ex: - raise ParseSQLResultError(str(ex)) - elif equals_ignore_case(ds.type, 'kingbase'): - with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: + for col_idx, col_name in enumerate(columns): + is_numeric = False try: - cursor.execute(sql) - res = cursor.fetchall() - columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for - field in - cursor.description] - fields_info = build_fields_info_from_cursor(cursor, origin_column, 'postgresql') - result_list = [ - {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} - except Exception as ex: - raise ParseSQLResultError(str(ex)) - elif equals_ignore_case(ds.type, 'es'): - try: - res, raw_columns = get_es_data_by_http(conf, sql) - columns = [field.get('name') for field in raw_columns] if origin_column else [field.get('name').lower() - for - field in - raw_columns] - fields_info = build_fields_info_from_es(raw_columns, origin_column) - result_list = [ - {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} - except Exception as ex: - raise Exception(str(ex)) - elif equals_ignore_case(ds.type, 'hive'): - with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: - try: - # Hive uses backticks for identifiers; normalize quoted identifiers as a compatibility fallback. - hive_sql = re.sub(r'"([A-Za-z_][A-Za-z0-9_]*)"', r'`\1`', sql) - cursor.execute(hive_sql) - res = cursor.fetchall() - columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for - field in - cursor.description] - fields_info = build_fields_info_from_cursor(cursor, origin_column, 'hive') - result_list = [ - {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in - res - ] - return {"fields": columns, "data": result_list, "fields_info": fields_info, - "sql": bytes.decode(base64.b64encode(bytes(hive_sql, 'utf-8')))} - except Exception as ex: - raise ParseSQLResultError(str(ex)) + type_code = result.cursor.description[col_idx][1] + is_numeric = is_numeric_type_code(type_code, dialect_name) + if is_mssql and is_mssql_string_type_code(type_code): + string_cols.append(col_idx) + except (IndexError, AttributeError): + pass + + fields_info.append({ + "name": col_name, + "is_numeric": is_numeric + }) + + string_col_idx_set = set(string_cols) + + res = _fetch_rows(result, max_rows) + result_list = [ + {str(columns[i]): convert_mssql_value(convert_value(value), i, string_col_idx_set, is_mssql) + for i, value in enumerate(tuple_item)} for tuple_item in + res + ] + return {"fields": columns, "data": result_list, "fields_info": fields_info, "sql": _encode_sql(sql)} + except Exception as ex: + raise ParseSQLResultError(str(ex)) + + +def _hive_sql(sql: str) -> str: + # Hive uses backticks for identifiers; normalize quoted identifiers as a compatibility fallback. + return re.sub(r'"([A-Za-z_][A-Za-z0-9_]*)"', r'`\1`', sql) + + +# py_driver 数据源的执行差异:(字段类型码族, SQL 预处理, 是否传驱动级超时)。 +# 新增 py_driver 数据源时在此登记一行,不必复制执行流程。 +_DRIVER_EXEC_PROFILES = { + 'dm': ('dm', None, True), + 'doris': ('mysql', None, False), + 'starrocks': ('mysql', None, False), + 'redshift': ('postgresql', None, False), + 'kingbase': ('postgresql', None, False), + 'hive': ('hive', _hive_sql, False), +} + + +def _exec_driver(ds, sql: str, origin_column, max_rows: Optional[int]): + conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) + ds_key = (ds.type or '').lower() + if ds_key == 'es': + try: + res, raw_columns = get_es_data_by_http(conf, sql) + if max_rows is not None: + res = res[:max_rows] + columns = [field.get('name') for field in raw_columns] if origin_column else [field.get('name').lower() + for field in raw_columns] + fields_info = build_fields_info_from_es(raw_columns, origin_column) + result_list = [ + {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in res + ] + return {"fields": columns, "data": result_list, "fields_info": fields_info, "sql": _encode_sql(sql)} + except Exception as ex: + raise Exception(str(ex)) + + profile = _DRIVER_EXEC_PROFILES.get(ds_key) + if profile is None: + # 与旧实现一致:未登记的 py_driver 类型不执行 + return None + type_family, prepare_sql, driver_timeout = profile + exec_sql_text = prepare_sql(sql) if prepare_sql else sql + with get_driver_pool(ds).connection() as conn, conn.cursor() as cursor: + try: + if driver_timeout: + cursor.execute(exec_sql_text, timeout=conf.timeout) + else: + cursor.execute(exec_sql_text) + res = _fetch_rows(cursor, max_rows) + columns = [field[0] for field in cursor.description] if origin_column else [field[0].lower() for + field in + cursor.description] + fields_info = build_fields_info_from_cursor(cursor, origin_column, type_family) + result_list = [ + {str(columns[i]): convert_value(value) for i, value in enumerate(tuple_item)} for tuple_item in res + ] + return {"fields": columns, "data": result_list, "fields_info": fields_info, + "sql": _encode_sql(exec_sql_text)} + except Exception as ex: + raise ParseSQLResultError(str(ex)) def build_fields_info_from_cursor(cursor, origin_column, db_type='postgresql'): @@ -1138,58 +1136,34 @@ def build_fields_info_from_es(raw_columns, origin_column): def get_sqlglot_dialect(ds_type: str) -> str: - """根据数据源类型获取 sqlglot dialect""" - if equals_ignore_case(ds_type, 'mysql', 'doris', 'starrocks'): - return 'mysql' - elif equals_ignore_case(ds_type, 'sqlServer'): - return 'tsql' - elif equals_ignore_case(ds_type, 'hive'): - return 'hive' - return None - - -# 通用危险函数(适用于所有数据库) -COMMON_DANGEROUS_FUNCTIONS = {'version', 'current_user', 'user', 'database'} - -# 特定数据库的危险函数 -# key 必须与 DB 枚举的 type 值一致(见 apps/db/constant.py),例如 PostgreSQL 的 type 是 'pg', -# 而不是 'postgresql';写成后者会让该数据源的危险函数规则整条失效。 + """根据数据源类型获取 sqlglot dialect(来自 apps/db/dialect 的方言策略注册表)""" + policy = dialect_registry.find(ds_type) + return policy.sqlglot_dialect if policy else None + + +# 通用危险函数(适用于所有数据库);规则本体在 apps/db/sql_guard/basic.py +COMMON_DANGEROUS_FUNCTIONS = set(sql_guard_basic.COMMON_DANGEROUS_FUNCTIONS) + +# 特定数据库的危险函数,由方言策略注册表派生(新增数据源类型请注册 DialectPolicy, +# 不要直接改这里)。key 与 DB 枚举的 type 值一致(小写)。 DS_SPECIFIC_DANGEROUS_FUNCTIONS = { - 'mysql': {'LOAD_FILE', 'INTO OUTFILE', 'INTO DUMPFILE'}, - 'doris': {'LOAD_FILE', 'INTO OUTFILE', 'INTO DUMPFILE'}, - 'starrocks': {'LOAD_FILE', 'INTO OUTFILE', 'INTO DUMPFILE'}, - 'pg': { - # 服务端文件读写族:只读连接也可能借此读到数据库主机的本地文件 - 'pg_read_file', - 'pg_read_binary_file', - 'pg_ls_dir', - 'pg_stat_file', - 'pg_write_file', - 'lo_import', - 'lo_export', - }, - 'sqlserver': {'EXEC', 'xp_cmdshell', 'sp_executesql'}, - 'oracle': {'UTL_FILE', 'DBMS_PIPE', 'DBMS_LOCK'}, - 'hive': {'ADD FILE', 'ADD JAR'}, + key: set(dialect_registry.get(key).dangerous_functions) + for key in sorted(dialect_registry.types()) + if dialect_registry.get(key).dangerous_functions } # 危险模式正则表达式(用于检查特殊语法) import re -DANGEROUS_PATTERNS = [ - r'\bINTO\s+OUTFILE\b', - r'\bINTO\s+DUMPFILE\b', - r'\bEXEC\s*\(', - r'\bCOPY\s+.*\bTO\s+PROGRAM\b', -] +DANGEROUS_PATTERNS = list(sql_guard_basic.DANGEROUS_PATTERNS) def get_dangerous_functions(ds_type: str) -> set: """获取危险函数(通用 + 特定数据源)""" functions = COMMON_DANGEROUS_FUNCTIONS.copy() - ds_key = ds_type.lower() if ds_type else '' - if ds_key in DS_SPECIFIC_DANGEROUS_FUNCTIONS: - functions.update(DS_SPECIFIC_DANGEROUS_FUNCTIONS[ds_key]) + policy = dialect_registry.find(ds_type) + if policy: + functions.update(policy.dangerous_functions) return functions @@ -1206,70 +1180,25 @@ def check_dangerous_functions(statements: list, ds_type: str) -> bool: return True +# 规则拒绝(可展示给调用方)时返回 (False, reason);以下 code 表示无法完成检查, +# 沿用旧行为抛出 ValueError("Parse SQL Error: ...")。 +_GUARD_ERROR_CODES = {"parse_error", "check_error", "rewrite_error", "unsupported_dialect"} + + def check_sql_read(sql: str, ds: CoreDatasource | AssistantOutDsSchema) -> tuple[bool, str]: """ - 检查 SQL 是否为安全的只读查询 - 返回: (是否安全, 错误原因) + 检查 SQL 是否为安全的只读查询(apps/db/sql_guard 的 basic 规则集) + 返回: (是否安全, 错误原因);无法解析时抛出 ValueError """ try: - normalized_sql = sql.strip().lstrip("(").strip() - first_keyword = normalized_sql.split(None, 1)[0].upper() if normalized_sql else "" - - # 根据配置决定是否允许元数据查询 - if settings.SQLBOT_ALLOW_METADATA_QUERIES: - allowed_read_commands = {"SELECT", "WITH", "SHOW", "DESCRIBE", "DESC", "EXPLAIN"} - else: - allowed_read_commands = {"SELECT", "WITH"} - - denied_write_commands = { - "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER", - "TRUNCATE", "MERGE", "COPY", "REPLACE", "GRANT", "REVOKE", - "USE", "SET", "CALL" - } - - if not first_keyword: - raise ValueError("Parse SQL Error") - if first_keyword in denied_write_commands: - return False, f"Write operation '{first_keyword}' is not allowed" - - # 1. 使用正则检查特殊模式 - for pattern in DANGEROUS_PATTERNS: - if re.search(pattern, sql, re.IGNORECASE): - return False, f"SQL contains dangerous pattern: {pattern}" - - dialect = get_sqlglot_dialect(ds.type) - statements = sqlglot.parse(sql, dialect=dialect) - - if not statements: - raise ValueError("Parse SQL Error") - - # 2. 使用 sqlglot 检查函数调用 - dangerous_functions = get_dangerous_functions(ds.type) - dangerous_functions_upper = {f.upper() for f in dangerous_functions} - for stmt in statements: - if stmt: - for func in stmt.find_all(exp.Anonymous): - if func.name.upper() in dangerous_functions_upper: - return False, f"SQL contains dangerous function: {func.name}" - - # 3. 检查写操作类型 - write_types = ( - exp.Insert, exp.Update, exp.Delete, - exp.Create, exp.Drop, exp.Alter, - exp.Merge, exp.Copy - ) - - for stmt in statements: - if stmt is None: - continue - if isinstance(stmt, write_types): - return False, f"SQL contains write operation: {type(stmt).__name__}" - - if first_keyword not in allowed_read_commands: - return False, f"SQL command '{first_keyword}' is not allowed. Only SELECT and WITH are permitted" - + ctx = build_guard_context(sql, ds.type, allow_metadata_queries=settings.SQLBOT_ALLOW_METADATA_QUERIES) + get_guard_pipeline(GUARD_BASIC).run(ctx) return True, "" - + except SqlRejected as e: + if e.code in _GUARD_ERROR_CODES: + detail = e.reason + raise ValueError(detail if detail.startswith("Parse SQL Error: ") else f"Parse SQL Error: {detail}") + return False, e.reason except Exception as e: raise ValueError(f"Parse SQL Error: {e}") diff --git a/backend/apps/db/dialect/__init__.py b/backend/apps/db/dialect/__init__.py new file mode 100644 index 00000000..b2690c21 --- /dev/null +++ b/backend/apps/db/dialect/__init__.py @@ -0,0 +1,20 @@ +"""Datasource dialect policies (sqlglot dialect, denylists, timeouts). + +Importing this package registers the built-in policies. +""" + +from apps.db.dialect import builtin # noqa: F401 # register built-in policies +from apps.db.dialect.base import DialectPolicy, TimeoutApplier +from apps.db.dialect.registry import ( + DialectRegistry, + UnsupportedDialectError, + dialect_registry, +) + +__all__ = [ + "DialectPolicy", + "DialectRegistry", + "TimeoutApplier", + "UnsupportedDialectError", + "dialect_registry", +] diff --git a/backend/apps/db/dialect/base.py b/backend/apps/db/dialect/base.py new file mode 100644 index 00000000..a156bbc9 --- /dev/null +++ b/backend/apps/db/dialect/base.py @@ -0,0 +1,48 @@ +"""Per-datasource SQL dialect policy. + +A policy describes how SQLBot parses, guards and bounds SQL for one datasource +type. New datasource types are supported by registering a new policy (see +``registry.py``); guard and execution code only depends on this interface. +""" + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +# Applies a per-statement execution timeout on an open SQLAlchemy connection +# that is about to run one query. Returns an optional ``restore`` callable that +# undoes any session-level change before the connection goes back to the pool; +# appliers that only affect the current transaction return ``None``. +TimeoutApplier = Callable[[Any, int], Callable[[], None] | None] + + +@dataclass(frozen=True) +class DialectPolicy: + # Datasource type code, must equal ``DB..type`` (case-insensitive). + ds_type: str + # sqlglot dialect used by the shared read-only check (``check_sql_read``). + # ``None`` is sqlglot's generic dialect. Kept as-is for compatibility: + # changing it changes how existing web-flow SQL is parsed. + sqlglot_dialect: str | None = None + # Precise sqlglot dialect used to parse and *render* SQL that SQLBot + # rewrites (strict profile). ``None`` falls back to ``sqlglot_dialect``. + precise_dialect: str | None = None + # Extra dangerous functions (case-insensitive names) on top of the common set. + dangerous_functions: frozenset[str] = field(default_factory=frozenset) + # Functions that can stall or exhaust the database (sleep, benchmark...). + # Enforced by strict guard profiles only. + resource_functions: frozenset[str] = field(default_factory=frozenset) + # Whether the strict guard (external agent SQL) supports this datasource. + agent_supported: bool = True + # Whether a row cap may be added by rewriting the SQL (LIMIT/TOP/FETCH). + # When ``False`` the cap is enforced only while fetching rows. + limit_rewrite: bool = True + # How to apply a per-statement timeout; ``None`` means "not supported". + apply_timeout: TimeoutApplier | None = None + + def key(self) -> str: + return self.ds_type.lower() + + @property + def strict_dialect(self) -> str | None: + return self.precise_dialect or self.sqlglot_dialect diff --git a/backend/apps/db/dialect/builtin.py b/backend/apps/db/dialect/builtin.py new file mode 100644 index 00000000..e132ef84 --- /dev/null +++ b/backend/apps/db/dialect/builtin.py @@ -0,0 +1,88 @@ +"""Built-in dialect policies for every ``DB`` datasource type. + +Adding a datasource type means appending one ``DialectPolicy`` here (or +registering one from another module); guard and execution code stays unchanged. + +``sqlglot_dialect`` keeps the parser each type used before this registry +existed (shared read-only check). ``precise_dialect`` is used where SQLBot +re-renders SQL, so the output must be valid for the real database. +""" + +import logging + +from apps.db.dialect.base import DialectPolicy +from apps.db.dialect.registry import DialectRegistry, dialect_registry + +logger = logging.getLogger(__name__) + +_MYSQL_FILE = frozenset({"LOAD_FILE", "INTO OUTFILE", "INTO DUMPFILE"}) +_PG_FILE = frozenset({ + # Server-side file access: even a read-only role may read host files. + "pg_read_file", + "pg_read_binary_file", + "pg_ls_dir", + "pg_stat_file", + "pg_write_file", + "lo_import", + "lo_export", +}) +_ORACLE_FILE = frozenset({"UTL_FILE", "DBMS_PIPE", "DBMS_LOCK"}) +_MYSQL_RESOURCE = frozenset({"sleep", "benchmark", "get_lock"}) +_PG_RESOURCE = frozenset({ + "pg_sleep", "pg_sleep_for", "pg_sleep_until", + "pg_advisory_lock", "pg_advisory_xact_lock", "pg_terminate_backend", "pg_cancel_backend", + "dblink", "dblink_exec", "dblink_connect", +}) +_ORACLE_RESOURCE = frozenset({"DBMS_LOCK", "DBMS_SESSION", "UTL_HTTP", "UTL_TCP", "UTL_INADDR"}) +_MSSQL_RESOURCE = frozenset({"WAITFOR", "OPENROWSET", "OPENDATASOURCE", "OPENQUERY"}) +_CK_RESOURCE = frozenset({"sleep", "sleepEachRow", "url", "remote", "remoteSecure", "file", "s3"}) +_HIVE_RESOURCE = frozenset({"reflect", "java_method"}) + + +def _pg_statement_timeout(conn, seconds: int): + # SET LOCAL only lasts until the current transaction ends, so a pooled + # connection does not keep the setting. + conn.exec_driver_sql(f"SET LOCAL statement_timeout = {int(seconds) * 1000}") + return None + + +def _mysql_execution_time(conn, seconds: int): + # max_execution_time is session scoped: remember and restore it so the + # pooled connection behaves as before for other callers. + previous = conn.exec_driver_sql("SELECT @@SESSION.max_execution_time").scalar() + conn.exec_driver_sql(f"SET SESSION max_execution_time = {int(seconds) * 1000}") + + def restore(): + conn.exec_driver_sql(f"SET SESSION max_execution_time = {int(previous or 0)}") + + return restore + + +BUILTIN_POLICIES = ( + DialectPolicy("mysql", "mysql", "mysql", _MYSQL_FILE, _MYSQL_RESOURCE, apply_timeout=_mysql_execution_time), + DialectPolicy("doris", "mysql", "doris", _MYSQL_FILE, _MYSQL_RESOURCE), + DialectPolicy("starrocks", "mysql", "starrocks", _MYSQL_FILE, _MYSQL_RESOURCE), + DialectPolicy("sqlServer", "tsql", "tsql", frozenset({"EXEC", "xp_cmdshell", "sp_executesql"}), + _MSSQL_RESOURCE), + DialectPolicy("hive", "hive", "hive", frozenset({"ADD FILE", "ADD JAR"}), _HIVE_RESOURCE), + DialectPolicy("pg", None, "postgres", _PG_FILE, _PG_RESOURCE, apply_timeout=_pg_statement_timeout), + DialectPolicy("excel", None, "postgres", _PG_FILE, _PG_RESOURCE, apply_timeout=_pg_statement_timeout), + DialectPolicy("kingbase", None, "postgres", _PG_FILE, _PG_RESOURCE), + DialectPolicy("redshift", None, "redshift", _PG_FILE, _PG_RESOURCE), + # Oracle < 12c has no FETCH FIRST; DM syntax only partly matches sqlglot's + # Oracle dialect. Cap rows while fetching instead of rewriting. + DialectPolicy("oracle", None, "oracle", _ORACLE_FILE, _ORACLE_RESOURCE, limit_rewrite=False), + DialectPolicy("dm", None, "oracle", _ORACLE_FILE, _ORACLE_RESOURCE, limit_rewrite=False), + DialectPolicy("ck", None, "clickhouse", frozenset(), _CK_RESOURCE), + # Elasticsearch SQL is executed over HTTP, not through sqlglot/SQLAlchemy. + DialectPolicy("es", None, None, frozenset(), frozenset(), agent_supported=False, limit_rewrite=False), +) + + +def register_builtin_policies(registry: DialectRegistry = dialect_registry) -> None: + for policy in BUILTIN_POLICIES: + if registry.find(policy.ds_type) is None: + registry.register(policy) + + +register_builtin_policies() diff --git a/backend/apps/db/dialect/registry.py b/backend/apps/db/dialect/registry.py new file mode 100644 index 00000000..69bd2e15 --- /dev/null +++ b/backend/apps/db/dialect/registry.py @@ -0,0 +1,41 @@ +"""Registry of dialect policies keyed by datasource type.""" + +import threading + +from apps.db.dialect.base import DialectPolicy + + +class UnsupportedDialectError(ValueError): + """Raised when no policy is registered for a datasource type.""" + + +class DialectRegistry: + def __init__(self) -> None: + self._policies: dict[str, DialectPolicy] = {} + self._lock = threading.Lock() + + def register(self, policy: DialectPolicy, *, replace: bool = False) -> DialectPolicy: + key = policy.key() + with self._lock: + if key in self._policies and not replace: + raise ValueError(f"Dialect policy already registered: {policy.ds_type}") + self._policies[key] = policy + return policy + + def find(self, ds_type: str | None) -> DialectPolicy | None: + if not ds_type: + return None + return self._policies.get(ds_type.lower()) + + def get(self, ds_type: str | None) -> DialectPolicy: + """Return the policy or fail closed when the type is unknown.""" + policy = self.find(ds_type) + if policy is None: + raise UnsupportedDialectError(f"Unsupported datasource type: {ds_type}") + return policy + + def types(self) -> set[str]: + return set(self._policies) + + +dialect_registry = DialectRegistry() diff --git a/backend/apps/db/sql_guard/__init__.py b/backend/apps/db/sql_guard/__init__.py new file mode 100644 index 00000000..40712d1d --- /dev/null +++ b/backend/apps/db/sql_guard/__init__.py @@ -0,0 +1,26 @@ +"""SQL guard: composable checks and rewriters run before SQL reaches a datasource.""" + +from apps.db.sql_guard.base import ( + GuardContext, + GuardPipeline, + SqlCheck, + SqlRejected, + SqlRewriter, + TableAccess, + build_context, +) +from apps.db.sql_guard.profiles import AGENT, BASIC, get_pipeline, register_profile + +__all__ = [ + "AGENT", + "BASIC", + "GuardContext", + "GuardPipeline", + "SqlCheck", + "SqlRejected", + "SqlRewriter", + "TableAccess", + "build_context", + "get_pipeline", + "register_profile", +] diff --git a/backend/apps/db/sql_guard/base.py b/backend/apps/db/sql_guard/base.py new file mode 100644 index 00000000..f06efdeb --- /dev/null +++ b/backend/apps/db/sql_guard/base.py @@ -0,0 +1,158 @@ +"""Core types of the SQL guard: context, check/rewrite protocols, pipeline. + +A guard run takes one SQL string for one datasource and either returns the SQL +to execute or raises ``SqlRejected``. Rules are independent ``SqlCheck`` and +``SqlRewriter`` objects composed into a ``GuardPipeline``; adding a rule means +adding an object to a profile, not editing the pipeline. + +Every failure is fatal: an unknown dialect, a parse error or an exception in +any rule rejects the SQL. Nothing is skipped silently. +""" + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +import sqlglot +from sqlglot import expressions as exp + +from apps.db.dialect import DialectPolicy, dialect_registry + + +class SqlRejected(ValueError): + """The SQL must not be executed. ``reason`` is safe to show to the caller.""" + + def __init__(self, reason: str, *, code: str = "rejected"): + super().__init__(reason) + self.reason = reason + self.code = code + + +@dataclass +class TableAccess: + """What the caller may read from one physical table (by lower-cased name).""" + + name: str + # Visible column names (lower-cased); ``None`` means every column is visible. + visible_columns: frozenset[str] | None = None + # All checked column names known for the table (lower-cased), used to + # attribute unqualified columns. Empty means unknown (treated strictly). + all_columns: frozenset[str] = frozenset() + # SQL boolean expression restricting rows, already escaped by its producer + # and written in the datasource's own dialect. + row_filter: str | None = None + # Schema the table belongs to (lower-cased); empty means "not checked". + schema: str = "" + + +@dataclass +class GuardContext: + sql: str + policy: DialectPolicy + # Physical tables the caller may read; ``None`` disables table checks. + tables: dict[str, TableAccess] | None = None + allow_metadata_queries: bool = False + max_rows: int | None = None + # Parse with ``policy.strict_dialect`` instead of the legacy dialect. + strict: bool = False + # Free-form values rules may share (e.g. flags for the response). + notes: dict[str, Any] = field(default_factory=dict) + _statements: list | None = field(default=None, repr=False) + + @property + def dialect(self) -> str | None: + return self.policy.strict_dialect if self.strict else self.policy.sqlglot_dialect + + def statements(self) -> list: + """Parse lazily; parse failures reject the SQL.""" + if self._statements is None: + try: + parsed = sqlglot.parse(self.sql, read=self.dialect) + except Exception as e: # sqlglot raises several error types + raise SqlRejected(f"Parse SQL Error: {e}", code="parse_error") from e + if not parsed: + raise SqlRejected("Parse SQL Error", code="parse_error") + # ``sqlglot.parse`` yields ``None`` for empty segments (e.g. a lone + # ``;``). They are dropped here; ``tree()`` enforces exactly one + # real statement where that matters. + self._statements = [stmt for stmt in parsed if stmt is not None] + return self._statements + + def tree(self) -> exp.Expression: + statements = self.statements() + if len(statements) != 1: + raise SqlRejected("Only a single SQL statement is allowed", code="multiple_statements") + return statements[0] + + def replace_sql(self, sql: str) -> None: + self.sql = sql + self._statements = None + + +@runtime_checkable +class SqlCheck(Protocol): + name: str + # ``True``: also run on the rewritten SQL. Checks about what the *caller* + # wrote (column permissions) set ``False`` because rewriters introduce + # constructs (``SELECT *`` wrappers) the caller is not allowed to write. + recheck: bool + + def check(self, ctx: GuardContext) -> None: + """Raise ``SqlRejected`` when the SQL must not run.""" + + +@runtime_checkable +class SqlRewriter(Protocol): + name: str + + def rewrite(self, ctx: GuardContext) -> str: + """Return the SQL to continue with (may be unchanged).""" + + +class GuardPipeline: + """Runs all checks, then rewriters, then re-runs ``recheck`` checks.""" + + def __init__(self, checks: Sequence[SqlCheck] = (), rewriters: Sequence[SqlRewriter] = ()): + self.checks = tuple(checks) + self.rewriters = tuple(rewriters) + + def extend(self, checks: Iterable[SqlCheck] = (), rewriters: Iterable[SqlRewriter] = ()) -> "GuardPipeline": + """Return a new pipeline with extra rules appended (this one is unchanged).""" + return GuardPipeline(self.checks + tuple(checks), self.rewriters + tuple(rewriters)) + + @staticmethod + def _run_checks(ctx: GuardContext, checks: Sequence[SqlCheck]) -> None: + for rule in checks: + try: + rule.check(ctx) + except SqlRejected: + raise + except Exception as e: + raise SqlRejected(f"SQL check '{rule.name}' failed: {e}", code="check_error") from e + + def run(self, ctx: GuardContext) -> str: + self._run_checks(ctx, self.checks) + if not self.rewriters: + return ctx.sql + for rule in self.rewriters: + try: + new_sql = rule.rewrite(ctx) + except SqlRejected: + raise + except Exception as e: + raise SqlRejected(f"SQL rewrite '{rule.name}' failed: {e}", code="rewrite_error") from e + if not new_sql: + raise SqlRejected(f"SQL rewrite '{rule.name}' produced no SQL", code="rewrite_error") + if new_sql != ctx.sql: + ctx.replace_sql(new_sql) + self._run_checks(ctx, [c for c in self.checks if getattr(c, "recheck", True)]) + return ctx.sql + + +def build_context(sql: str, ds_type: str | None, **kwargs) -> GuardContext: + """Create a context; unknown datasource types are rejected (fail closed).""" + try: + policy = dialect_registry.get(ds_type) + except ValueError as e: + raise SqlRejected(str(e), code="unsupported_dialect") from e + return GuardContext(sql=sql, policy=policy, **kwargs) diff --git a/backend/apps/db/sql_guard/basic.py b/backend/apps/db/sql_guard/basic.py new file mode 100644 index 00000000..7a75606a --- /dev/null +++ b/backend/apps/db/sql_guard/basic.py @@ -0,0 +1,120 @@ +"""Read-only checks shared by every SQL execution path. + +These rules keep the behaviour ``check_sql_read`` had before it was split into +a pipeline. Each rule is independent so profiles can reuse or extend them. +""" + +import re +from collections.abc import Iterable + +from sqlglot import expressions as exp + +from apps.db.sql_guard.base import GuardContext, SqlRejected + +# Applies to every datasource type, on top of ``DialectPolicy.dangerous_functions``. +COMMON_DANGEROUS_FUNCTIONS = frozenset({"version", "current_user", "user", "database"}) + +DANGEROUS_PATTERNS = ( + r"\bINTO\s+OUTFILE\b", + r"\bINTO\s+DUMPFILE\b", + r"\bEXEC\s*\(", + r"\bCOPY\s+.*\bTO\s+PROGRAM\b", +) + +READ_COMMANDS = frozenset({"SELECT", "WITH"}) +METADATA_COMMANDS = frozenset({"SHOW", "DESCRIBE", "DESC", "EXPLAIN"}) +DENIED_COMMANDS = frozenset({ + "INSERT", "UPDATE", "DELETE", "CREATE", "DROP", "ALTER", + "TRUNCATE", "MERGE", "COPY", "REPLACE", "GRANT", "REVOKE", + "USE", "SET", "CALL", +}) +WRITE_STATEMENT_TYPES = ( + exp.Insert, exp.Update, exp.Delete, + exp.Create, exp.Drop, exp.Alter, + exp.Merge, exp.Copy, +) + + +def first_keyword(sql: str) -> str: + normalized = sql.strip().lstrip("(").strip() + return normalized.split(None, 1)[0].upper() if normalized else "" + + +def dangerous_functions_for(ctx: GuardContext) -> frozenset[str]: + return COMMON_DANGEROUS_FUNCTIONS | ctx.policy.dangerous_functions + + +class LeadingKeywordCheck: + """Reject empty SQL and SQL that starts with a write/admin keyword.""" + + name = "leading_keyword" + recheck = True + + def check(self, ctx: GuardContext) -> None: + keyword = first_keyword(ctx.sql) + if not keyword: + raise SqlRejected("Parse SQL Error", code="parse_error") + if keyword in DENIED_COMMANDS: + raise SqlRejected(f"Write operation '{keyword}' is not allowed", code="write_operation") + + +class DangerousPatternCheck: + name = "dangerous_pattern" + recheck = True + + def __init__(self, patterns: Iterable[str] = DANGEROUS_PATTERNS): + self.patterns = tuple(patterns) + + def check(self, ctx: GuardContext) -> None: + for pattern in self.patterns: + if re.search(pattern, ctx.sql, re.IGNORECASE): + raise SqlRejected(f"SQL contains dangerous pattern: {pattern}", code="dangerous_pattern") + + +class AnonymousFunctionCheck: + """Reject denylisted functions that sqlglot parses as ``Anonymous`` calls.""" + + name = "anonymous_function" + recheck = True + + def check(self, ctx: GuardContext) -> None: + denied = {f.upper() for f in dangerous_functions_for(ctx)} + for stmt in ctx.statements(): + for func in stmt.find_all(exp.Anonymous): + if func.name.upper() in denied: + raise SqlRejected(f"SQL contains dangerous function: {func.name}", code="dangerous_function") + + +class WriteStatementCheck: + name = "write_statement" + recheck = True + + def check(self, ctx: GuardContext) -> None: + for stmt in ctx.statements(): + if isinstance(stmt, WRITE_STATEMENT_TYPES): + raise SqlRejected(f"SQL contains write operation: {type(stmt).__name__}", code="write_operation") + + +class ReadCommandCheck: + name = "read_command" + recheck = True + + def check(self, ctx: GuardContext) -> None: + allowed = READ_COMMANDS | (METADATA_COMMANDS if ctx.allow_metadata_queries else frozenset()) + keyword = first_keyword(ctx.sql) + if keyword not in allowed: + raise SqlRejected( + f"SQL command '{keyword}' is not allowed. Only SELECT and WITH are permitted", + code="command_not_allowed", + ) + + +def basic_checks() -> list: + """Order matches the legacy ``check_sql_read`` so reasons stay the same.""" + return [ + LeadingKeywordCheck(), + DangerousPatternCheck(), + AnonymousFunctionCheck(), + WriteStatementCheck(), + ReadCommandCheck(), + ] diff --git a/backend/apps/db/sql_guard/profiles.py b/backend/apps/db/sql_guard/profiles.py new file mode 100644 index 00000000..83caa730 --- /dev/null +++ b/backend/apps/db/sql_guard/profiles.py @@ -0,0 +1,56 @@ +"""Named guard profiles. + +``basic`` is what every SQL execution path enforces (``check_sql_read``). +``agent`` is used for SQL written by an external caller: basic checks, strict +checks, then row permissions and a row cap. + +Register a new profile (or extend one) with ``register_profile``; callers pick +profiles by name so they never assemble rules themselves. +""" + +import threading +from collections.abc import Callable + +from apps.db.sql_guard.base import GuardPipeline +from apps.db.sql_guard.basic import basic_checks +from apps.db.sql_guard.rewriters import RowLimitRewriter, RowPermissionRewriter +from apps.db.sql_guard.strict import strict_checks + +PipelineFactory = Callable[[], GuardPipeline] + +BASIC = "basic" +AGENT = "agent" + +_profiles: dict[str, PipelineFactory] = {} +_lock = threading.Lock() + + +def register_profile(name: str, factory: PipelineFactory, *, replace: bool = False) -> None: + with _lock: + if name in _profiles and not replace: + raise ValueError(f"Guard profile already registered: {name}") + _profiles[name] = factory + + +def get_pipeline(name: str) -> GuardPipeline: + factory = _profiles.get(name) + if factory is None: + raise ValueError(f"Unknown guard profile: {name}") + return factory() + + +def _basic() -> GuardPipeline: + return GuardPipeline(basic_checks()) + + +def _agent() -> GuardPipeline: + # Callers must build the context with ``strict=True`` so SQL is parsed and + # re-rendered with the datasource's precise dialect. + return GuardPipeline( + basic_checks() + strict_checks(), + rewriters=[RowPermissionRewriter(), RowLimitRewriter()], + ) + + +register_profile(BASIC, _basic) +register_profile(AGENT, _agent) diff --git a/backend/apps/db/sql_guard/rewriters.py b/backend/apps/db/sql_guard/rewriters.py new file mode 100644 index 00000000..8b6e8407 --- /dev/null +++ b/backend/apps/db/sql_guard/rewriters.py @@ -0,0 +1,77 @@ +"""Deterministic SQL rewriters (no model involved).""" + +import sqlglot +from sqlglot import expressions as exp + +from apps.db.sql_guard.base import GuardContext, SqlRejected +from apps.db.sql_guard.tables import physical_tables_in, scopes_of + + +class RowPermissionRewriter: + """Replace each restricted table with ``(SELECT * FROM t WHERE ) AS ``. + + Filters come from the permission engine (values already escaped, written in + the datasource dialect). Every occurrence is wrapped, including tables + inside CTEs and subqueries, and the original alias is kept so column + references stay valid. + """ + + name = "row_permission" + + def rewrite(self, ctx: GuardContext) -> str: + if not ctx.tables or not any(a.row_filter for a in ctx.tables.values()): + return ctx.sql + tree = ctx.tree().copy() + targets = [] + for scope in scopes_of(tree): + for table in physical_tables_in(scope): + access = ctx.tables.get(table.name) + if access is not None and access.row_filter: + targets.append((table, access)) + if not targets: + return ctx.sql + for table, access in targets: + try: + condition = sqlglot.parse_one(access.row_filter, read=ctx.dialect, into=exp.Condition) + except Exception as e: + raise SqlRejected(f"Row permission for '{table.name}' cannot be applied", code="row_permission") from e + alias = table.node.args.get("alias") + alias_ident = alias.this.copy() if alias is not None else table.node.this.copy() + source = table.node.copy() + source.set("alias", None) + subquery = exp.select("*").from_(source).where(condition).subquery(alias=alias_ident) + table.node.replace(subquery) + ctx.notes["row_permission_applied"] = sorted({t.name for t, _ in targets}) + return tree.sql(dialect=ctx.dialect) + + +class RowLimitRewriter: + """Cap the number of returned rows at ``ctx.max_rows + 1``. + + One extra row lets the caller report truncation. An existing smaller literal + limit is kept; a larger one (or none) is replaced with the cap. Dialects + whose policy disables ``limit_rewrite`` are capped while fetching instead. + """ + + name = "row_limit" + + def rewrite(self, ctx: GuardContext) -> str: + if not ctx.max_rows or not ctx.policy.limit_rewrite: + return ctx.sql + cap = int(ctx.max_rows) + 1 + tree = ctx.tree().copy() + current = self._current_limit(tree) + if current is not None and current <= cap: + return ctx.sql + return tree.limit(cap).sql(dialect=ctx.dialect) + + @staticmethod + def _current_limit(tree: exp.Expression): + node = tree.args.get("limit") + if node is None: + return None + value = node.args.get("expression") or node.args.get("count") + if isinstance(value, exp.Literal) and value.is_int: + return int(value.this) + # Non-literal limits (parameters, expressions) are replaced. + return None diff --git a/backend/apps/db/sql_guard/strict.py b/backend/apps/db/sql_guard/strict.py new file mode 100644 index 00000000..ac6da3d3 --- /dev/null +++ b/backend/apps/db/sql_guard/strict.py @@ -0,0 +1,209 @@ +"""Strict rules for SQL submitted by an external caller (agent tools). + +These run on top of the basic read-only checks. They assume the SQL author is +untrusted: one statement, no locks or nested writes, no resource-exhausting +functions, and only the tables/columns the caller is allowed to read. +""" + +from sqlglot import expressions as exp + +from apps.db.sql_guard.base import GuardContext, SqlRejected +from apps.db.sql_guard.basic import WRITE_STATEMENT_TYPES, dangerous_functions_for +from apps.db.sql_guard.tables import physical_tables_in, scopes_of + +# Identity/introspection helpers sqlglot parses into named nodes rather than +# ``Anonymous``; the basic check only sees the latter. +NAMED_IDENTITY_FUNCTIONS = ( + exp.CurrentUser, exp.CurrentVersion, exp.CurrentDatabase, exp.CurrentSchema, + exp.CurrentCatalog, exp.CurrentRole, exp.SessionUser, +) + +# Catalog schemas that expose the whole database structure. +SYSTEM_SCHEMAS = frozenset({ + "information_schema", "pg_catalog", "mysql", "performance_schema", "sys", + "sysibm", "system", "dba", "all", "user", +}) + + +def _function_names(node: exp.Expression) -> set[str]: + """Every name a function call can be matched by: bare, qualified and package.""" + names = set() + if isinstance(node, exp.Anonymous): + names.add(node.name.upper()) + elif isinstance(node, exp.Func): + names.update(n.upper() for n in node.sql_names()) + names.add(type(node).__name__.upper()) + parent = node.parent + if isinstance(parent, exp.Dot) and parent.expression is node: + # ``PKG.FUNC(...)`` / ``schema.PKG.FUNC(...)``: match the package too. + for ident in parent.this.find_all(exp.Identifier): + names.add(ident.name.upper()) + return names + + +class AgentDialectCheck: + name = "agent_dialect" + recheck = False + + def check(self, ctx: GuardContext) -> None: + if not ctx.policy.agent_supported: + raise SqlRejected(f"Datasource type '{ctx.policy.ds_type}' is not supported", code="unsupported_dialect") + + +class SingleStatementCheck: + name = "single_statement" + recheck = True + + def check(self, ctx: GuardContext) -> None: + ctx.tree() + + +class NestedWriteCheck: + """Reject DML/DDL anywhere in the tree (e.g. ``WITH x AS (DELETE ...)``).""" + + name = "nested_write" + recheck = True + + def check(self, ctx: GuardContext) -> None: + for node in ctx.tree().walk(): + if isinstance(node, WRITE_STATEMENT_TYPES): + raise SqlRejected(f"SQL contains write operation: {type(node).__name__}", code="write_operation") + + +class LockAndIntoCheck: + """Reject row locks, ``SELECT ... INTO`` and table lock hints.""" + + name = "lock_and_into" + recheck = True + + def check(self, ctx: GuardContext) -> None: + tree = ctx.tree() + for select in tree.find_all(exp.Select): + if select.args.get("locks"): + raise SqlRejected("Locking clauses (FOR UPDATE/SHARE) are not allowed", code="locking") + if select.args.get("into"): + raise SqlRejected("SELECT ... INTO is not allowed", code="select_into") + if any(True for _ in tree.find_all(exp.WithTableHint)): + raise SqlRejected("Table hints are not allowed", code="table_hint") + + +class FunctionDenylistCheck: + """Denylisted, identity and resource-exhausting functions, in any node form.""" + + name = "function_denylist" + recheck = True + + def check(self, ctx: GuardContext) -> None: + denied = {f.upper() for f in dangerous_functions_for(ctx) | ctx.policy.resource_functions} + for node in ctx.tree().walk(): + if isinstance(node, NAMED_IDENTITY_FUNCTIONS): + raise SqlRejected(f"SQL contains dangerous function: {node.sql_name()}", code="dangerous_function") + if isinstance(node, exp.Func): + hit = _function_names(node) & denied + if hit: + raise SqlRejected(f"SQL contains dangerous function: {sorted(hit)[0]}", code="dangerous_function") + + +class TableAllowlistCheck: + """Every physical table must be one the caller may read, in its own schema.""" + + name = "table_allowlist" + recheck = True + + def check(self, ctx: GuardContext) -> None: + if ctx.tables is None: + raise SqlRejected("No table permissions were resolved", code="table_access") + tables = [t for scope in scopes_of(ctx.tree()) for t in physical_tables_in(scope)] + if not tables: + raise SqlRejected("SQL must read from at least one table", code="table_access") + denied = set() + for table in tables: + access = ctx.tables.get(table.name) + if access is None or table.schema in SYSTEM_SCHEMAS or table.catalog in SYSTEM_SCHEMAS: + denied.add(table.node.sql(dialect=ctx.dialect)) + elif table.schema and access.schema and table.schema != access.schema: + denied.add(table.node.sql(dialect=ctx.dialect)) + if denied: + # Never list the allowed tables: that would disclose the schema. + raise SqlRejected(f"SQL references tables that are not accessible: {', '.join(sorted(denied))}", + code="table_access") + + +class ColumnPermissionCheck: + """Reject ``*`` on restricted tables and any reference to a hidden column. + + Hidden columns are rejected everywhere (select list, WHERE, JOIN, ORDER BY, + function arguments) because filters can reveal their values too. + Unqualified columns are checked against every restricted table in the same + scope; ambiguity is resolved conservatively. + """ + + name = "column_permission" + recheck = False + + def check(self, ctx: GuardContext) -> None: + if ctx.tables is None: + raise SqlRejected("No table permissions were resolved", code="table_access") + for scope in scopes_of(ctx.tree()): + restricted = {} + for table in physical_tables_in(scope): + access = ctx.tables.get(table.name) + if access is not None and access.visible_columns is not None: + restricted[table.alias] = access + if not restricted: + continue + self._check_stars(scope, restricted) + self._check_columns(scope, restricted) + + @staticmethod + def _check_stars(scope, restricted) -> None: + select = scope.expression + if not isinstance(select, exp.Select): + return + for projection in select.expressions: + if isinstance(projection, exp.Star): + raise SqlRejected("SELECT * is not allowed on tables with column permissions; list the columns", + code="column_access") + if isinstance(projection, exp.Column) and isinstance(projection.this, exp.Star): + if projection.table.lower() in restricted: + raise SqlRejected(f"{projection.table}.* is not allowed on tables with column permissions", + code="column_access") + + @staticmethod + def _is_hidden(access, name: str, qualified: bool) -> bool: + if name in access.visible_columns: + return False + if qualified or not access.all_columns: + # ``t.col`` must be visible on ``t``; with no column catalogue an + # unqualified name cannot be attributed, so it is treated as hidden. + return True + # Unqualified: only names that really exist on the table are hidden. + # Others belong to another table or are select-list aliases. + return name in access.all_columns + + @classmethod + def _check_columns(cls, scope, restricted) -> None: + for column in scope.columns: + name = column.name.lower() + if not name or isinstance(column.this, exp.Star): + continue + qualifier = column.table.lower() + if qualifier: + candidates = [restricted[qualifier]] if qualifier in restricted else [] + else: + candidates = list(restricted.values()) + for access in candidates: + if cls._is_hidden(access, name, qualified=bool(qualifier)): + raise SqlRejected(f"Column '{column.name}' is not accessible", code="column_access") + + +def strict_checks() -> list: + return [ + AgentDialectCheck(), + SingleStatementCheck(), + NestedWriteCheck(), + LockAndIntoCheck(), + FunctionDenylistCheck(), + TableAllowlistCheck(), + ColumnPermissionCheck(), + ] diff --git a/backend/apps/db/sql_guard/tables.py b/backend/apps/db/sql_guard/tables.py new file mode 100644 index 00000000..cc8ee6fe --- /dev/null +++ b/backend/apps/db/sql_guard/tables.py @@ -0,0 +1,68 @@ +"""Physical table discovery shared by checks and rewriters. + +A physical table is a ``Table`` node that does not resolve to a CTE or derived +table in its scope. CTEs may shadow physical names +(``WITH t AS (SELECT * FROM t)``); scope resolution keeps the inner reference +physical, unlike name-based filtering. + +Scopes are built once per call: ``Scope`` objects from different +``traverse_scope`` runs are not comparable. +""" + +from collections.abc import Iterator +from dataclasses import dataclass + +from sqlglot import expressions as exp +from sqlglot.optimizer.scope import Scope, traverse_scope + +from apps.db.sql_guard.base import GuardContext, SqlRejected + + +@dataclass(frozen=True) +class PhysicalTable: + node: exp.Table + scope: Scope + + @property + def name(self) -> str: + return self.node.name.lower() + + @property + def alias(self) -> str: + return self.node.alias_or_name.lower() + + @property + def schema(self) -> str: + return (self.node.db or "").lower() + + @property + def catalog(self) -> str: + return (self.node.catalog or "").lower() + + +def scopes_of(tree: exp.Expression) -> list[Scope]: + try: + return traverse_scope(tree) + except Exception as e: # sqlglot raises OptimizeError and others + raise SqlRejected(f"Unable to analyse SQL structure: {e}", code="parse_error") from e + + +def physical_tables_in(scope: Scope) -> list[PhysicalTable]: + found = [] + for table in scope.tables: + if not table.name: + # Table-valued functions (``FROM generate_series(...)``) have no name. + raise SqlRejected("Table functions are not allowed", code="table_function") + if isinstance(scope.sources.get(table.alias_or_name), Scope) and not table.args.get("db"): + continue # resolves to a CTE or derived table in this scope + found.append(PhysicalTable(table, scope)) + return found + + +def iter_physical_tables(tree: exp.Expression) -> Iterator[PhysicalTable]: + for scope in scopes_of(tree): + yield from physical_tables_in(scope) + + +def physical_table_names(ctx: GuardContext) -> set[str]: + return {t.name for t in iter_physical_tables(ctx.tree())} diff --git a/backend/tests/test_sql_guard.py b/backend/tests/test_sql_guard.py new file mode 100644 index 00000000..7b3c1f97 --- /dev/null +++ b/backend/tests/test_sql_guard.py @@ -0,0 +1,278 @@ +"""Offline regressions for the shared SQL guard and the dialect registry. + +``check_sql_read`` now delegates to the ``basic`` guard profile; the ``agent`` +profile adds strict checks and deterministic rewrites for SQL written by an +external caller. No database, model or network is used. +""" + +from types import SimpleNamespace + +import pytest +import sqlbot_xpack # noqa: F401 # Initialize the published wheel before app imports. +import sqlglot + +from apps.db.constant import DB +from apps.db.db import ( + DS_SPECIFIC_DANGEROUS_FUNCTIONS, + check_sql_read, + get_dangerous_functions, + get_sqlglot_dialect, +) +from apps.db.dialect import ( + DialectPolicy, + DialectRegistry, + UnsupportedDialectError, + dialect_registry, +) +from apps.db.sql_guard import ( + AGENT, + GuardPipeline, + SqlRejected, + TableAccess, + build_context, + get_pipeline, +) + +EMP = TableAccess( + "emp", + visible_columns=frozenset({"id", "name", "dept_id"}), + all_columns=frozenset({"id", "name", "dept_id", "salary"}), + row_filter="(\"region\" IN ('east'))", + schema="public", +) +DEPT = TableAccess("dept", all_columns=frozenset({"id", "title"})) +TABLES = {"emp": EMP, "dept": DEPT} + + +def _ds(ds_type): + return SimpleNamespace(type=ds_type) + + +def agent(sql, ds_type="pg", tables=TABLES, max_rows=200): + ctx = build_context(sql, ds_type, tables=tables, max_rows=max_rows, strict=True) + return get_pipeline(AGENT).run(ctx), ctx + + +def rejected(sql, ds_type="pg", **kwargs): + with pytest.raises(SqlRejected) as info: + agent(sql, ds_type, **kwargs) + return info.value + + +# --- dialect registry ------------------------------------------------------- + +def test_every_db_type_has_a_policy(): + assert {db.type.lower() for db in DB} <= dialect_registry.types() + + +def test_legacy_sqlglot_dialects_are_unchanged(): + # The shared read-only check must keep parsing web-flow SQL as before. + legacy = {"mysql": "mysql", "doris": "mysql", "starrocks": "mysql", "sqlServer": "tsql", "hive": "hive"} + for db in DB: + assert get_sqlglot_dialect(db.type) == legacy.get(db.type) + + +def test_unknown_type_fails_closed(): + assert get_sqlglot_dialect("nope") is None + with pytest.raises(UnsupportedDialectError): + dialect_registry.get("nope") + with pytest.raises(SqlRejected) as info: + build_context("SELECT 1", "nope") + assert info.value.code == "unsupported_dialect" + + +def test_pg_family_shares_the_file_function_denylist(): + for ds_type in ("pg", "excel", "kingbase", "redshift"): + assert {"pg_read_file", "lo_import"} <= get_dangerous_functions(ds_type) + assert ds_type in DS_SPECIFIC_DANGEROUS_FUNCTIONS + is_safe, reason = check_sql_read("SELECT pg_read_file('/etc/passwd')", _ds(ds_type)) + assert not is_safe and "dangerous function" in reason + + +def test_a_new_dialect_is_one_registration(): + registry = DialectRegistry() + registry.register(DialectPolicy("newdb", "postgres", resource_functions=frozenset({"stall"}))) + assert registry.get("NEWDB").sqlglot_dialect == "postgres" + with pytest.raises(ValueError): + registry.register(DialectPolicy("newdb")) + + +# --- basic profile (check_sql_read) ------------------------------------------ + +@pytest.mark.parametrize("sql", ["SELECT id FROM users", "WITH t AS (SELECT 1 AS x) SELECT x FROM t"]) +def test_reads_are_allowed(sql): + assert check_sql_read(sql, _ds("pg")) == (True, "") + + +@pytest.mark.parametrize("sql,reason", [ + ("DROP TABLE users", "Write operation 'DROP' is not allowed"), + ("SELECT * INTO OUTFILE '/tmp/x' FROM users", "dangerous pattern"), + ("SELECT user()", "dangerous function"), + ("SHOW TABLES", "is not allowed. Only SELECT and WITH are permitted"), +]) +def test_basic_rejections_keep_their_reasons(sql, reason): + is_safe, message = check_sql_read(sql, _ds("pg")) + assert not is_safe + assert reason in message + + +@pytest.mark.parametrize("sql", ["", " "]) +def test_unparseable_sql_still_raises(sql): + with pytest.raises(ValueError, match="Parse SQL Error"): + check_sql_read(sql, _ds("pg")) + + +# --- agent profile: structure ------------------------------------------------- + +@pytest.mark.parametrize("sql,code", [ + ("SELECT name FROM emp; SELECT 1", "multiple_statements"), + ("WITH x AS (DELETE FROM emp RETURNING *) SELECT * FROM x", "write_operation"), + ("SELECT name FROM emp FOR UPDATE", "locking"), + ("SELECT name FROM generate_series(1, 3)", "table_function"), + ("SELECT 1", "table_access"), +]) +def test_agent_rejects_unsafe_structure(sql, code): + assert rejected(sql).code == code + + +def test_agent_rejects_table_hints_and_select_into(): + assert rejected("SELECT name FROM emp WITH (UPDLOCK)", "sqlServer").code == "table_hint" + assert rejected("SELECT name INTO copy FROM emp").code in {"select_into", "write_operation"} + + +@pytest.mark.parametrize("ds_type,sql", [ + ("pg", "SELECT pg_sleep(5), name FROM emp"), + ("pg", "SELECT version(), name FROM emp"), + ("pg", "SELECT current_user, name FROM emp"), + ("pg", "SELECT dblink('x', 'y'), name FROM emp"), + ("mysql", "SELECT sleep(1), name FROM emp"), + ("mysql", "SELECT BENCHMARK(10, MD5('a')), name FROM emp"), + ("oracle", "SELECT DBMS_LOCK.SLEEP(5), name FROM emp"), +]) +def test_agent_rejects_dangerous_functions_in_any_node_form(ds_type, sql): + assert rejected(sql, ds_type).code == "dangerous_function" + + +@pytest.mark.parametrize("ds_type", ["es", "nope"]) +def test_agent_rejects_unsupported_datasources(ds_type): + assert rejected("SELECT name FROM emp", ds_type).code == "unsupported_dialect" + + +# --- agent profile: tables ---------------------------------------------------- + +@pytest.mark.parametrize("sql", [ + "SELECT * FROM secret", + "SELECT * FROM information_schema.tables", + "SELECT name FROM information_schema.emp", + "SELECT name FROM other.emp", + "SELECT title FROM dept WHERE id IN (SELECT id FROM secret)", +]) +def test_agent_rejects_tables_outside_the_allowlist(sql): + error = rejected(sql) + assert error.code == "table_access" + # The allowed table names must not be disclosed. + assert "dept" not in error.reason.replace("FROM dept", "") + + +def test_cte_shadowing_a_table_still_reaches_the_real_table(): + error = rejected("WITH secret AS (SELECT * FROM secret) SELECT * FROM secret") + assert error.code == "table_access" + + +def test_schema_qualified_allowed_table_is_accepted(): + sql, _ = agent("SELECT name FROM public.emp") + assert "public.emp" in sql + + +# --- agent profile: columns --------------------------------------------------- + +@pytest.mark.parametrize("sql", [ + "SELECT * FROM emp", + "SELECT e.* FROM emp e", + "SELECT e.salary FROM emp e", + "SELECT name FROM emp WHERE salary > 100", + "SELECT name FROM emp ORDER BY salary", + "SELECT max(salary) FROM emp", + "WITH c AS (SELECT * FROM emp) SELECT name FROM c", +]) +def test_hidden_columns_are_rejected_everywhere(sql): + assert rejected(sql).code == "column_access" + + +@pytest.mark.parametrize("sql", [ + "SELECT name FROM emp", + "SELECT count(*) FROM emp", + "SELECT * FROM dept", + "SELECT name AS label FROM emp ORDER BY label", + "SELECT e.name, d.title FROM emp e JOIN dept d ON e.dept_id = d.id", +]) +def test_visible_columns_are_allowed(sql): + agent(sql) + + +# --- agent profile: rewrites -------------------------------------------------- + +def test_row_filter_wraps_every_occurrence_and_keeps_aliases(): + sql, ctx = agent("SELECT e.name FROM emp e WHERE e.dept_id IN (SELECT dept_id FROM emp)") + assert sql.count("""WHERE ("region" IN ('east'))""") == 2 + assert ") AS e " in sql + assert ctx.notes["row_permission_applied"] == ["emp"] + + +def test_rewritten_sql_is_valid_and_has_no_unfiltered_reference(): + sql, _ = agent("SELECT d.title, count(*) FROM emp JOIN dept d ON emp.dept_id = d.id GROUP BY d.title") + tree = sqlglot.parse_one(sql, read="postgres") + emp_refs = [t for t in tree.find_all(sqlglot.exp.Table) if t.name == "emp"] + assert emp_refs and all(t.find_ancestor(sqlglot.exp.Subquery) for t in emp_refs) + + +def test_unparseable_row_filter_rejects_instead_of_running_unfiltered(): + broken = {"emp": TableAccess("emp", row_filter="((")} + assert rejected("SELECT name FROM emp", tables=broken).code in {"row_permission", "rewrite_error"} + + +@pytest.mark.parametrize("ds_type,sql,expected", [ + ("pg", "SELECT title FROM dept", "LIMIT 201"), + ("pg", "SELECT title FROM dept LIMIT 5", "LIMIT 5"), + ("mysql", "SELECT title FROM dept LIMIT 100000", "LIMIT 201"), + ("sqlServer", "SELECT title FROM dept", "TOP 201"), +]) +def test_row_cap(ds_type, sql, expected): + out, _ = agent(sql, ds_type) + assert expected in out + + +def test_oracle_family_is_capped_while_fetching_not_rewritten(): + out, _ = agent("SELECT title FROM dept", "oracle") + assert "FETCH" not in out.upper() and "ROWNUM" not in out.upper() + + +# --- extensibility ------------------------------------------------------------ + +def test_extra_rules_plug_in_without_changing_the_pipeline(): + class DenyTitle: + name = "deny_title" + recheck = True + + def check(self, ctx): + if "title" in ctx.sql.lower(): + raise SqlRejected("no titles", code="custom") + + pipeline = get_pipeline(AGENT).extend(checks=[DenyTitle()]) + ctx = build_context("SELECT title FROM dept", "pg", tables=TABLES, strict=True) + with pytest.raises(SqlRejected, match="no titles"): + pipeline.run(ctx) + + +def test_a_crashing_rule_rejects_instead_of_being_skipped(): + class Broken: + name = "broken" + recheck = True + + def check(self, ctx): + raise RuntimeError("boom") + + ctx = build_context("SELECT title FROM dept", "pg", tables=TABLES) + with pytest.raises(SqlRejected) as info: + GuardPipeline([Broken()]).run(ctx) + assert info.value.code == "check_error" From 8ebbfab2028ee26a8469d419cd81e0e50e19f55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A2=81=E6=B7=8B=E6=B6=9B?= Date: Mon, 16 Nov 2026 11:07:13 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(mcp):=20=E6=96=B0=E5=A2=9E=E6=97=A0?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=9A=84=20sqlbot-mcp=20=E5=85=A5=E5=8F=A3?= =?UTF-8?q?=E4=B8=8E=20Agent=20=E4=BB=A4=E7=89=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在 MCP 进程(8001)新增 /sqlbot-mcp,使用官方 mcp SDK 的 Streamable HTTP(无状态),外部 Agent 自行编写 SQL,SQLBot 不调用模型 - 提供 5 个工具:agent_list_datasources、agent_get_query_context、agent_list_tables、agent_describe_tables、agent_execute_sql;执行结果写入会话记录(来源 MCP),成功与拒绝均写入操作日志 - agent_execute_sql 使用 agent 校验规则集:表/列权限、确定性行权限改写、语句超时与行数上限;会话记录只保存 Agent 原始 SQL - 新增 Agent 令牌(sys_agent_token,迁移 076):用户自助创建,格式 sk-sqlbot-,绑定创建者与工作空间,可限定数据源,库中只存 SHA-256 摘要;令牌只在 /sqlbot-mcp 生效 - 鉴权、限流、审计、工具注册均为可替换扩展点;旧的 /mcp 不变 - 前端个人菜单新增“Agent 令牌”管理弹窗,五种语言文案;同步更新 CONTEXT.md、docs/agents 与 backend/README.md AI-Coding: true AI-Tool: TophantAI --- CONTEXT.md | 6 +- backend/README.md | 20 + .../alembic/versions/076_agent_token_ddl.py | 49 +++ backend/apps/api.py | 3 +- backend/apps/mcp/sqlbot_mcp/__init__.py | 5 + backend/apps/mcp/sqlbot_mcp/audit.py | 50 +++ backend/apps/mcp/sqlbot_mcp/auth.py | 164 +++++++ backend/apps/mcp/sqlbot_mcp/ratelimit.py | 81 ++++ backend/apps/mcp/sqlbot_mcp/server.py | 106 +++++ backend/apps/mcp/sqlbot_mcp/service.py | 316 +++++++++++++ backend/apps/mcp/sqlbot_mcp/tools/__init__.py | 17 + backend/apps/mcp/sqlbot_mcp/tools/base.py | 79 ++++ backend/apps/mcp/sqlbot_mcp/tools/context.py | 31 ++ .../apps/mcp/sqlbot_mcp/tools/datasources.py | 18 + backend/apps/mcp/sqlbot_mcp/tools/execute.py | 35 ++ backend/apps/mcp/sqlbot_mcp/tools/tables.py | 41 ++ backend/apps/system/api/agent_token.py | 105 +++++ .../apps/system/crud/agent_token_manage.py | 208 +++++++++ backend/apps/system/models/system_model.py | 22 + backend/common/audit/models/log_model.py | 3 + backend/locales/en.json | 14 +- backend/locales/ko-KR.json | 14 +- backend/locales/th-TH.json | 14 +- backend/locales/zh-CN.json | 14 +- backend/locales/zh-TW.json | 14 +- backend/main.py | 15 +- backend/tests/test_agent_token.py | 160 +++++++ backend/tests/test_sqlbot_mcp.py | 179 ++++++++ backend/tests/test_sqlbot_mcp_integration.py | 216 +++++++++ docs/agents/backend.md | 2 + docs/agents/security.md | 23 + frontend/src/api/agentToken.ts | 34 ++ frontend/src/components/layout/AgentToken.vue | 415 ++++++++++++++++++ frontend/src/components/layout/Person.vue | 15 + frontend/src/i18n/en.json | 22 + frontend/src/i18n/ko-KR.json | 22 + frontend/src/i18n/th-TH.json | 22 + frontend/src/i18n/zh-CN.json | 22 + frontend/src/i18n/zh-TW.json | 22 + 39 files changed, 2590 insertions(+), 8 deletions(-) create mode 100644 backend/alembic/versions/076_agent_token_ddl.py create mode 100644 backend/apps/mcp/sqlbot_mcp/__init__.py create mode 100644 backend/apps/mcp/sqlbot_mcp/audit.py create mode 100644 backend/apps/mcp/sqlbot_mcp/auth.py create mode 100644 backend/apps/mcp/sqlbot_mcp/ratelimit.py create mode 100644 backend/apps/mcp/sqlbot_mcp/server.py create mode 100644 backend/apps/mcp/sqlbot_mcp/service.py create mode 100644 backend/apps/mcp/sqlbot_mcp/tools/__init__.py create mode 100644 backend/apps/mcp/sqlbot_mcp/tools/base.py create mode 100644 backend/apps/mcp/sqlbot_mcp/tools/context.py create mode 100644 backend/apps/mcp/sqlbot_mcp/tools/datasources.py create mode 100644 backend/apps/mcp/sqlbot_mcp/tools/execute.py create mode 100644 backend/apps/mcp/sqlbot_mcp/tools/tables.py create mode 100644 backend/apps/system/api/agent_token.py create mode 100644 backend/apps/system/crud/agent_token_manage.py create mode 100644 backend/tests/test_agent_token.py create mode 100644 backend/tests/test_sqlbot_mcp.py create mode 100644 backend/tests/test_sqlbot_mcp_integration.py create mode 100644 frontend/src/api/agentToken.ts create mode 100644 frontend/src/components/layout/AgentToken.vue diff --git a/CONTEXT.md b/CONTEXT.md index 900ed86c..d042c18e 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -153,9 +153,13 @@ _Avoid_: App Secret(验签密钥,不是透传凭据) _Avoid_: app_id(仅用于反查应用,无认证作用)、Assistant Certificate **MCP**: -以独立服务进程暴露问数工具集的集成形态,调用者以真实用户身份接入,数据权限按该用户计算。 +以独立服务进程暴露问数工具集的集成形态,调用者以真实用户身份接入,数据权限按该用户计算。MCP 服务进程上有两个入口:`/mcp` 把问数流程整体暴露出去,由 SQLBot 的模型生成 SQL;`/sqlbot-mcp` 是无模型入口,由外部 Agent 自己写 SQL,SQLBot 只提供上下文和受控执行。 _Avoid_: Ordinary Assistant、Page-embedded Assistant +**Agent Token / Agent 令牌**: +用户自助创建、供外部 Agent 调用 `/sqlbot-mcp` 的 Bearer 凭据(`sk-sqlbot-` 前缀),绑定创建者和一个工作空间,可再收窄到部分数据源;只能访问 `/sqlbot-mcp`,库中只存摘要。 +_Avoid_: API Key(可调用全部 API 的 AK/SK 签名凭据) + **Dashboard / 仪表板**: 由会话图表快照与文本、Tab 组件组成的画布;图表组件是会话记录的配置快照,数据在查看时实时查询。 _Avoid_: Chat、chat record diff --git a/backend/README.md b/backend/README.md index f0c99ce2..699ecfb2 100644 --- a/backend/README.md +++ b/backend/README.md @@ -14,6 +14,8 @@ SQLBOT_SKIP_TABLE_CHECK=true 开启后仅跳过表白名单检查,可能允许查询数据源账号可读取的未勾选表。真实表名提取始终执行,无法提取表名时仍拒绝 SQL。SQL 只读、危险函数、元数据查询控制、行权限处理及行数限制仍按原有流程执行。 +该开关只影响网页问数和 `/mcp` 的模型生成 SQL;`/sqlbot-mcp` 的外部提交 SQL 始终做表和列检查。 + ## 数据源存储密钥 默认无需配置。API 首次启动时会在持久化目录 `UPLOAD_DIR/.secrets/datasource-keys.json` 自动生成部署专用密钥,默认完整路径为 `/opt/sqlbot/data/file/.secrets/datasource-keys.json`。保留现有 `file` 数据卷,API 和 MCP 使用同一文件。若数据库已有新版密文而密钥文件缺失,启动会停止,请恢复原密钥文件。 @@ -36,3 +38,21 @@ python3 -c 'import base64,json,secrets; key=base64.b64encode(secrets.token_bytes 轮换时先停止该部署的 API 和 MCP,保留旧密钥,加入新密钥并将 `active` 切换到新标识,原子替换文件。启动 API,等待现有配置重新加密,再恢复其他进程。用当前密钥文件验证数据库备份恢复后,可从在线文件移除旧密钥。历史备份所需密钥应离线保留至备份过期。恢复时先准备匹配该备份的密钥文件,再启动 API。已暴露的旧 ECB 密文仍可解密,应轮换对应数据源自身的凭据。 浏览器写入仍兼容旧请求编码,部署通过 HTTPS 保护传输。数据源列表和写入响应省略配置,授权编辑详情仅提供表单字段,密码和额外连接参数留空表示沿用。连接目标变化时需要重新提供已保存的敏感连接字段。API 客户端可发送显式空字符串清除密码或额外连接参数。 + +## sqlbot-mcp(无模型 Agent 入口) + +`/sqlbot-mcp` 让 Claude、Codex 等外部 Agent 自己写 SQL,SQLBot 不调用模型,只提供数据源、表结构、术语、SQL 示例,并在行列权限、只读校验、语句超时和行数上限下执行。它和 `/mcp` 跑在同一个 MCP 进程上(`uvicorn main:mcp_app --port 8001`)。 + +1. 在网页右上角个人菜单打开“Agent 令牌”,创建令牌(可限定数据源)。令牌只显示一次,以创建者身份和当前工作空间生效。 +2. 在 Agent 中添加 MCP(Streamable HTTP): + + ```bash + claude mcp add --transport http sqlbot-mcp http://:8001/sqlbot-mcp \ + --header "Authorization: Bearer sk-sqlbot-..." + ``` + + 只支持 stdio 的客户端可用 `npx mcp-remote http://:8001/sqlbot-mcp --header "Authorization: Bearer sk-sqlbot-..."` 桥接。 + +工具:`agent_list_datasources`、`agent_get_query_context`、`agent_list_tables`、`agent_describe_tables`、`agent_execute_sql`。执行记录写入创建者的会话历史(来源为 MCP),调用写入操作日志。 + +部署注意:8001 端口应只对内网或可信网络开放;按令牌限流使用 `security_config` 的 `rate_limit_per_minute`(进程内计数,多 worker 时每个 worker 单独计数);语句超时目前在 PostgreSQL / Excel / MySQL 生效,其他数据源依赖连接配置的超时;数据源连接账号建议只授予只读权限。 diff --git a/backend/alembic/versions/076_agent_token_ddl.py b/backend/alembic/versions/076_agent_token_ddl.py new file mode 100644 index 00000000..4336e1da --- /dev/null +++ b/backend/alembic/versions/076_agent_token_ddl.py @@ -0,0 +1,49 @@ +"""076_agent_token_ddl + +Bearer tokens for the SQLBot MCP server used by external agents. Only a +SHA-256 digest of each token is stored. + +Revision ID: 3b7e0c1f9a42 +Revises: d91543ca7b21 +Create Date: 2026-11-13 18:00:00 + +""" +import sqlalchemy as sa +import sqlmodel.sql.sqltypes + +from alembic import op + +# revision identifiers, used by Alembic. +revision = '3b7e0c1f9a42' +down_revision = 'd91543ca7b21' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'sys_agent_token', + sa.Column('id', sa.BigInteger(), nullable=False), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column('uid', sa.BigInteger(), nullable=False), + sa.Column('oid', sa.BigInteger(), nullable=False), + sa.Column('token_prefix', sqlmodel.sql.sqltypes.AutoString(length=32), nullable=False), + sa.Column('token_hash', sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), + sa.Column('status', sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column('allowed_ds_ids', sa.Text(), nullable=True), + sa.Column('create_time', sa.BigInteger(), nullable=False, server_default='0'), + sa.Column('last_used_time', sa.BigInteger(), nullable=True), + sa.Column('last_used_ip', sqlmodel.sql.sqltypes.AutoString(length=64), nullable=True), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index(op.f('ix_sys_agent_token_id'), 'sys_agent_token', ['id'], unique=False) + op.create_index(op.f('ix_sys_agent_token_uid'), 'sys_agent_token', ['uid'], unique=False) + op.create_index(op.f('ix_sys_agent_token_token_prefix'), 'sys_agent_token', ['token_prefix'], unique=True) + + +def downgrade(): + # Dropping the table revokes every issued agent token. + op.drop_index(op.f('ix_sys_agent_token_token_prefix'), table_name='sys_agent_token') + op.drop_index(op.f('ix_sys_agent_token_uid'), table_name='sys_agent_token') + op.drop_index(op.f('ix_sys_agent_token_id'), table_name='sys_agent_token') + op.drop_table('sys_agent_token') diff --git a/backend/apps/api.py b/backend/apps/api.py index ec5a4209..05d9d68a 100644 --- a/backend/apps/api.py +++ b/backend/apps/api.py @@ -5,7 +5,7 @@ from apps.data_training.api import data_training from apps.datasource.api import datasource, table_relation, recommended_problem from apps.mcp import mcp -from apps.system.api import login, user, aimodel, workspace, assistant, parameter, apikey, variable_api +from apps.system.api import login, user, aimodel, workspace, assistant, parameter, apikey, variable_api, agent_token from apps.terminology.api import terminology from apps.settings.api import base #from audit.api import audit_api @@ -27,6 +27,7 @@ api_router.include_router(table_relation.router) api_router.include_router(parameter.router) api_router.include_router(apikey.router) +api_router.include_router(agent_token.router) api_router.include_router(recommended_problem.router) diff --git a/backend/apps/mcp/sqlbot_mcp/__init__.py b/backend/apps/mcp/sqlbot_mcp/__init__.py new file mode 100644 index 00000000..a3cb44b3 --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/__init__.py @@ -0,0 +1,5 @@ +"""``sqlbot-mcp``: model-free MCP tools for external agents. + +Agents write SQL themselves; SQLBot provides datasources, schema, business +knowledge and guarded execution. See ``server.py`` for assembly. +""" diff --git a/backend/apps/mcp/sqlbot_mcp/audit.py b/backend/apps/mcp/sqlbot_mcp/audit.py new file mode 100644 index 00000000..078710d6 --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/audit.py @@ -0,0 +1,50 @@ +"""Default audit sink: write tool calls to SQLBot's operation log (sys_logs). + +Query results and SQL text are never stored here; the chat record keeps the +SQL for the owner. Rejections are recorded too, so repeated denials from one +token stand out. +""" + +from datetime import datetime + +from sqlmodel import Session + +from apps.mcp.sqlbot_mcp.auth import AgentPrincipal +from apps.mcp.sqlbot_mcp.tools.base import AuditSink +from common.audit.models.log_model import ( + OperationModules, + OperationStatus, + OperationType, + SystemLog, +) +from common.core.db import engine +from common.utils.utils import SQLBotLogUtil + + +class SystemLogAuditSink(AuditSink): + def record(self, *, principal: AgentPrincipal, tool: str, ok: bool, duration_ms: int, + detail: str | None = None, resource_id: str | None = None) -> None: + user = principal.user + log = SystemLog( + operation_type=OperationType.EXECUTE_SQL.value if tool == "agent_execute_sql" else OperationType.VIEW.value, + operation_detail=f"sqlbot-mcp {tool}", + user_id=user.id, + user_name=getattr(user, "name", None) or getattr(user, "account", None), + operation_status=(OperationStatus.SUCCESS if ok else OperationStatus.FAILED).value, + ip_address=principal.client_ip, + execution_time=duration_ms, + error_message=(detail or "")[:1000] or None, + module=OperationModules.AGENT_MCP.value, + oid=principal.oid, + resource_id=resource_id, + request_method="MCP", + request_path=tool, + create_time=datetime.now(), + remark=f"{principal.credential_kind}:{principal.credential_id}", + ) + try: + with Session(engine) as session: + session.add(log) + session.commit() + except Exception: + SQLBotLogUtil.exception("[sqlbot-mcp] failed to write audit log") diff --git a/backend/apps/mcp/sqlbot_mcp/auth.py b/backend/apps/mcp/sqlbot_mcp/auth.py new file mode 100644 index 00000000..c214083a --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/auth.py @@ -0,0 +1,164 @@ +"""Authentication for the SQLBot MCP server. + +An ``Authenticator`` turns an incoming HTTP request into an ``AgentPrincipal`` +(or rejects it). Tools and services only depend on ``AgentPrincipal``, so a new +scheme (e.g. OAuth 2.1) is a new ``Authenticator`` and no tool changes. + +``AuthMiddleware`` runs the authenticator for every request to the MCP mount +and exposes the principal through a context variable; the value is inherited +by ``asyncio.to_thread`` workers. +""" + +import asyncio +import contextvars +from dataclasses import dataclass +from typing import Protocol + +from sqlmodel import Session, select +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Receive, Scope, Send + +from apps.system.crud import agent_token_manage as tokens +from apps.system.models.system_model import UserWsModel +from apps.system.models.user import UserModel +from apps.system.schemas.system_schema import UserInfoDTO +from common.core.db import engine +from common.utils.utils import SQLBotLogUtil + + +@dataclass(frozen=True) +class AgentPrincipal: + """Who is calling, and what their credential narrows access to.""" + + user: UserInfoDTO + credential_id: str + credential_kind: str + # ``None`` follows the user's datasource permissions. + allowed_ds_ids: frozenset[int] | None = None + client_ip: str | None = None + + @property + def oid(self) -> int: + return self.user.oid + + def allows_datasource(self, ds_id: int) -> bool: + return self.allowed_ds_ids is None or int(ds_id) in self.allowed_ds_ids + + +class AuthenticationFailed(Exception): + pass + + +class Authenticator(Protocol): + async def authenticate(self, request: Request) -> AgentPrincipal: + """Return the caller or raise ``AuthenticationFailed``.""" + + +_principal: contextvars.ContextVar[AgentPrincipal | None] = contextvars.ContextVar( + "sqlbot_mcp_principal", default=None) + + +def current_principal() -> AgentPrincipal: + principal = _principal.get() + if principal is None: + # Tools must never run without an authenticated caller. + raise PermissionError("Not authenticated") + return principal + + +def client_ip(request: Request) -> str | None: + for header in ("x-real-ip", "x-forwarded-for", "x-client-ip"): + value = request.headers.get(header) + if value: + return value.split(",")[0].strip()[:64] + return request.client.host if request.client else None + + +def bearer_token(request: Request) -> str | None: + header = request.headers.get("authorization", "") + scheme, _, value = header.partition(" ") + if scheme.lower() != "bearer" or not value.strip(): + return None + return value.strip() + + +class AgentTokenAuthenticator: + """``Authorization: Bearer sk-sqlbot-...`` tokens issued to a SQLBot user.""" + + kind = "agent_token" + + async def authenticate(self, request: Request) -> AgentPrincipal: + token = bearer_token(request) + if not token: + raise AuthenticationFailed("missing bearer token") + ip = client_ip(request) + verified, user = await asyncio.to_thread(self._verify, token, ip) + if verified is None or user is None: + raise AuthenticationFailed("invalid token") + return AgentPrincipal(user=user, credential_id=str(verified.token_id), credential_kind=self.kind, + allowed_ds_ids=verified.allowed_ds_ids, client_ip=ip) + + @staticmethod + def _verify(token: str, ip: str | None): + with Session(engine) as session: + verified = tokens.verify_token(session, token) + if verified is None: + return None, None + user = load_principal_user(session, verified.uid, verified.oid) + if user is None: + return None, None + try: + tokens.touch_token(session, verified.token_id, ip) + except Exception: + SQLBotLogUtil.exception("Failed to record agent token usage") + return verified, user + + +def load_principal_user(session: Session, uid: int, oid: int) -> UserInfoDTO | None: + """Build the caller's identity in workspace ``oid`` straight from the database. + + Mirrors ``get_user_info`` but is not cached, so disabling a user or removing + them from the workspace takes effect on the next tool call. The token is + bound to one workspace: the caller acts there, whatever workspace they last + selected in the web UI. + """ + db_user = session.get(UserModel, uid) + if db_user is None or db_user.status != 1: + return None + user = UserInfoDTO.model_validate(db_user.model_dump()) + user.isAdmin = user.id == 1 and user.account == "admin" + user.oid = oid + if user.isAdmin: + return user + membership = session.exec(select(UserWsModel).where(UserWsModel.uid == uid, UserWsModel.oid == oid)).first() + if membership is None: + return None + user.weight = membership.weight + return user + + +class AuthMiddleware: + """Reject unauthenticated requests before they reach the MCP transport.""" + + def __init__(self, app: ASGIApp, authenticator: Authenticator): + self.app = app + self.authenticator = authenticator + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + request = Request(scope, receive) + try: + principal = await self.authenticator.authenticate(request) + except AuthenticationFailed: + response = JSONResponse({"error": "invalid_token", "error_description": "Authentication required"}, + status_code=401, headers={"WWW-Authenticate": 'Bearer realm="sqlbot-mcp"'}) + await response(scope, receive, send) + return + token = _principal.set(principal) + try: + await self.app(scope, receive, send) + finally: + _principal.reset(token) diff --git a/backend/apps/mcp/sqlbot_mcp/ratelimit.py b/backend/apps/mcp/sqlbot_mcp/ratelimit.py new file mode 100644 index 00000000..320c09ae --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/ratelimit.py @@ -0,0 +1,81 @@ +"""Per-credential rate limiting for SQLBot MCP tools. + +``RateLimiter`` is a protocol so a shared backend (e.g. Redis) can replace the +in-process implementation when running several workers. +""" + +import threading +import time +from collections import deque +from dataclasses import dataclass +from typing import Protocol + + +class RateLimited(Exception): + pass + + +class Slot(Protocol): + def release(self) -> None: ... + + +class RateLimiter(Protocol): + def acquire(self, kind: str, key: str, *, heavy: bool = False) -> Slot: + """Reserve one call (and a concurrency slot when ``heavy``) or raise ``RateLimited``.""" + + +class _NoopSlot: + def release(self) -> None: + return None + + +@dataclass +class _Bucket: + calls: deque + running: int = 0 + + +class InMemoryRateLimiter: + """Sliding one-minute window per credential plus a cap on concurrent heavy calls.""" + + def __init__(self, per_minute: int = 60, max_concurrent_heavy: int = 2, *, enabled: bool = True, + clock=time.monotonic): + self.per_minute = per_minute + self.max_concurrent_heavy = max_concurrent_heavy + self.enabled = enabled + self._clock = clock + self._buckets: dict[tuple[str, str], _Bucket] = {} + self._lock = threading.Lock() + + def acquire(self, kind: str, key: str, *, heavy: bool = False) -> Slot: + if not self.enabled: + return _NoopSlot() + now = self._clock() + with self._lock: + bucket = self._buckets.setdefault((kind, key), _Bucket(deque())) + while bucket.calls and now - bucket.calls[0] >= 60: + bucket.calls.popleft() + if len(bucket.calls) >= self.per_minute: + raise RateLimited("Rate limit exceeded, retry later") + if heavy and bucket.running >= self.max_concurrent_heavy: + raise RateLimited("Too many queries running for this token, retry later") + bucket.calls.append(now) + if heavy: + bucket.running += 1 + return _HeavySlot(self, bucket) if heavy else _NoopSlot() + + def _release(self, bucket: _Bucket) -> None: + with self._lock: + bucket.running = max(0, bucket.running - 1) + + +class _HeavySlot: + def __init__(self, limiter: InMemoryRateLimiter, bucket: _Bucket): + self._limiter = limiter + self._bucket = bucket + self._released = False + + def release(self) -> None: + if not self._released: + self._released = True + self._limiter._release(self._bucket) diff --git a/backend/apps/mcp/sqlbot_mcp/server.py b/backend/apps/mcp/sqlbot_mcp/server.py new file mode 100644 index 00000000..a730afbb --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/server.py @@ -0,0 +1,106 @@ +"""Assemble the ``sqlbot-mcp`` server (model-free tools for external agents). + +``build_sqlbot_mcp`` wires the FastMCP instance, tools, authenticator, rate +limiter and audit sink. ``mount_sqlbot_mcp`` attaches it to an ASGI app at a +path and returns a lifespan context manager the host app must enter. + +Transport is MCP Streamable HTTP (stateless, JSON responses), so any worker +can serve any request. Swap or add parts by passing them in; nothing here is +hard-wired to a specific authenticator or backend. +""" + +import contextlib +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from mcp.server.fastmcp import FastMCP +from mcp.server.transport_security import TransportSecuritySettings +from starlette.types import ASGIApp, Receive, Scope, Send + +from apps.mcp.sqlbot_mcp.audit import SystemLogAuditSink +from apps.mcp.sqlbot_mcp.auth import ( + AgentTokenAuthenticator, + Authenticator, + AuthMiddleware, +) +from apps.mcp.sqlbot_mcp.ratelimit import InMemoryRateLimiter, RateLimiter +from apps.mcp.sqlbot_mcp.tools import TOOL_MODULES, register_tools +from apps.mcp.sqlbot_mcp.tools.base import AuditSink, ToolRuntime +from common.core.security_config import get_security_config + +SERVER_NAME = "sqlbot-mcp" +DEFAULT_MOUNT_PATH = "/sqlbot-mcp" +INSTRUCTIONS = ( + "SQLBot data access for agents. Typical flow: agent_list_datasources -> " + "agent_get_query_context -> write SQL -> agent_execute_sql. If a table is " + "missing from the context, use agent_list_tables and agent_describe_tables. " + "SQLBot never calls a model; you write the SQL." +) + + +@dataclass +class SqlbotMcp: + mcp: FastMCP + app: ASGIApp + + +def default_rate_limiter() -> RateLimiter: + config = get_security_config() + return InMemoryRateLimiter(per_minute=config.rate_limit_per_minute, enabled=config.enable_rate_limiting) + + +def build_sqlbot_mcp(*, authenticator: Authenticator | None = None, rate_limiter: RateLimiter | None = None, + audit: AuditSink | None = None, tool_modules=TOOL_MODULES, + transport_security: TransportSecuritySettings | None = None, + session_factory: Callable[[], Any] | None = None) -> SqlbotMcp: + mcp = FastMCP( + SERVER_NAME, + instructions=INSTRUCTIONS, + stateless_http=True, + json_response=True, + streamable_http_path="/", + # Host/Origin checks are done by the deployment (reverse proxy / + # network policy); requests here are already bearer-authenticated. + transport_security=transport_security or TransportSecuritySettings(enable_dns_rebinding_protection=False), + ) + runtime = ToolRuntime(rate_limiter=rate_limiter or default_rate_limiter(), audit=audit or SystemLogAuditSink()) + if session_factory is not None: + runtime.session_factory = session_factory + register_tools(mcp, runtime, tool_modules) + app = AuthMiddleware(mcp.streamable_http_app(), authenticator or AgentTokenAuthenticator()) + return SqlbotMcp(mcp=mcp, app=app) + + +class _TrailingSlash: + """Serve ``/sqlbot-mcp`` and ``/sqlbot-mcp/`` alike, without a redirect. + + A 307 to the slash form would answer unauthenticated requests before + ``AuthMiddleware`` runs and breaks clients that do not follow redirects. + """ + + def __init__(self, app: ASGIApp, path: str): + self.app = app + self.path = path.rstrip("/") + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] == "http" and scope.get("path") == self.path: + scope = dict(scope, path=self.path + "/", raw_path=(self.path + "/").encode()) + await self.app(scope, receive, send) + + +def mount_sqlbot_mcp(host, server: SqlbotMcp, path: str = DEFAULT_MOUNT_PATH): + """Mount on a Starlette/FastAPI ``host`` and return the lifespan to enter. + + Returns an async context manager running the MCP session manager; enter it + from the host's lifespan. + """ + host.mount(path, server.app) + host.add_middleware(_TrailingSlash, path=path) + + @contextlib.asynccontextmanager + async def lifespan(): + async with server.mcp.session_manager.run(): + yield + + return lifespan diff --git a/backend/apps/mcp/sqlbot_mcp/service.py b/backend/apps/mcp/sqlbot_mcp/service.py new file mode 100644 index 00000000..27e49c45 --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/service.py @@ -0,0 +1,316 @@ +"""Business logic behind the SQLBot MCP tools. + +Every function takes an ``AgentPrincipal`` and a database ``Session`` and is +synchronous; tools run them in a worker thread. No function here calls a model. +Datasource ids outside the caller's reach and missing ids raise the same +``ToolError`` so ids cannot be probed. +""" + +from dataclasses import dataclass +from typing import Any + +import orjson +from sqlmodel import Session, select + +from apps.chat.curd.chat import ( + create_chat, + finish_record, + save_error_message, + save_question, + save_sql, + save_sql_exec_data, +) +from apps.chat.models.chat_model import Chat, ChatQuestion, CreateChat +from apps.data_training.curd.data_training import get_training_template +from apps.datasource.crud.datasource import ( + get_datasource_list, + get_table_obj_by_ds, + get_table_schema, +) +from apps.datasource.crud.permission import get_row_permission_filters, is_normal_user +from apps.datasource.models.datasource import CoreDatasource +from apps.db.db import check_connection, exec_sql, get_version +from apps.db.sql_guard import ( + AGENT, + SqlRejected, + TableAccess, + build_context, + get_pipeline, +) +from apps.mcp.sqlbot_mcp.auth import AgentPrincipal +from apps.terminology.curd.terminology import ( + expand_with_terminology, + get_terminology_template, +) +from common.core.embedding_config import load_embedding_config +from common.utils.data_format import DataFormat +from common.utils.utils import SQLBotLogUtil, prepare_for_orjson + +DEFAULT_MAX_ROWS = 200 +MAX_ROWS_LIMIT = 1000 +STATEMENT_TIMEOUT_SECONDS = 30 +LIST_TABLES_LIMIT = 200 +# Chat.origin values: 0 page, 1 mcp, 2 assistant. +CHAT_ORIGIN_MCP = 1 + +SQL_WRITING_RULES = ( + "Write exactly one read-only SELECT (or WITH ... SELECT) statement for the given engine. " + "Use only tables and columns listed in `schema`; columns that are not listed are not accessible. " + "List columns explicitly instead of SELECT * on tables with restricted columns. " + "Do not use locking clauses, table functions or system catalogs. " + "Row-level permissions and a row cap are applied by SQLBot automatically." +) + + +class ToolError(Exception): + """Expected failure; ``message`` is safe to show to the agent.""" + + def __init__(self, message: str, *, code: str = "error"): + super().__init__(message) + self.message = message + self.code = code + + +NOT_FOUND = "Datasource not found or not accessible" + + +@dataclass +class ResolvedDatasource: + ds: CoreDatasource + engine: str + + +# --- access ------------------------------------------------------------------ + +def accessible_datasources(session: Session, principal: AgentPrincipal) -> list[CoreDatasource]: + datasources = get_datasource_list(session=session, user=principal.user) + return [ds for ds in datasources if principal.allows_datasource(ds.id)] + + +def resolve_datasource(session: Session, principal: AgentPrincipal, datasource_id: Any) -> ResolvedDatasource: + try: + ds_id = int(datasource_id) + except (TypeError, ValueError): + raise ToolError(NOT_FOUND, code="not_found") + if not principal.allows_datasource(ds_id): + raise ToolError(NOT_FOUND, code="not_found") + ds = session.get(CoreDatasource, ds_id) + if ds is None or ds.oid != principal.oid: + raise ToolError(NOT_FOUND, code="not_found") + engine_name = "PostgreSQL" if ds.type == "excel" else (ds.type_name or ds.type) + return ResolvedDatasource(ds=ds, engine=engine_name) + + +def _engine_with_version(resolved: ResolvedDatasource) -> str: + try: + return f"{resolved.engine} {get_version(resolved.ds)}".strip() + except Exception: + return resolved.engine + + +# --- list_datasources -------------------------------------------------------- + +def list_datasources(session: Session, principal: AgentPrincipal) -> list[dict]: + return [ + {"id": str(ds.id), "name": ds.name, "type": ds.type, + "engine": "PostgreSQL" if ds.type == "excel" else (ds.type_name or ds.type), + "description": ds.description or ""} + for ds in accessible_datasources(session, principal) + ] + + +# --- tables ------------------------------------------------------------------ + +def _table_comment(table) -> str: + return ((table.custom_comment or "").strip() or (table.table_comment or "").strip()) + + +def list_tables(session: Session, principal: AgentPrincipal, datasource_id: Any, + keyword: str | None = None) -> dict: + resolved = resolve_datasource(session, principal, datasource_id) + needle = (keyword or "").strip().lower() + items = [] + for obj in get_table_obj_by_ds(session=session, current_user=principal.user, ds=resolved.ds): + comment = _table_comment(obj.table) + if needle and needle not in obj.table.table_name.lower() and needle not in comment.lower(): + continue + items.append({"table_name": obj.table.table_name, "comment": comment}) + total = len(items) + return {"datasource_id": str(resolved.ds.id), "total": total, "truncated": total > LIST_TABLES_LIMIT, + "tables": items[:LIST_TABLES_LIMIT]} + + +def describe_tables(session: Session, principal: AgentPrincipal, datasource_id: Any, + table_names: list[str]) -> dict: + resolved = resolve_datasource(session, principal, datasource_id) + wanted = [str(name) for name in (table_names or []) if str(name).strip()] + if not wanted: + raise ToolError("table_names must not be empty", code="invalid_argument") + schema, found = get_table_schema(session, principal.user, resolved.ds, "", embedding=False, table_list=wanted) + found_set = {name.lower() for name in found} + return { + "datasource_id": str(resolved.ds.id), + "engine": _engine_with_version(resolved), + "schema": schema, + "tables": found, + # Missing and forbidden tables look the same. + "not_accessible": [name for name in wanted if name.lower() not in found_set], + } + + +# --- query context ----------------------------------------------------------- + +def _custom_prompt(session: Session, principal: AgentPrincipal, ds_id: int) -> str: + try: + from sqlbot_xpack.custom_prompt.curd.custom_prompt import find_custom_prompts + from sqlbot_xpack.custom_prompt.models.custom_prompt_model import ( + CustomPromptTypeEnum, + ) + from sqlbot_xpack.license.license_manage import SQLBotLicenseUtil + except Exception: # pragma: no cover - xpack is a hard dependency + return "" + if not SQLBotLicenseUtil.valid(): + return "" + prompt, _ = find_custom_prompts(session, CustomPromptTypeEnum.GENERATE_SQL, principal.oid, ds_id) + return prompt or "" + + +def get_query_context(session: Session, principal: AgentPrincipal, datasource_id: Any, question: str, + keywords: str | None = None) -> dict: + question = (question or "").strip() + if not question: + raise ToolError("question must not be empty", code="invalid_argument") + resolved = resolve_datasource(session, principal, datasource_id) + ds = resolved.ds + config = load_embedding_config(session) + keywords = ",".join(k.strip() for k in (keywords or "").split(",") if k.strip()) or None + expanded = expand_with_terminology(keywords, session, principal.oid, ds.id) if keywords else None + + schema, tables = get_table_schema(session, principal.user, ds, question, keywords=expanded, config=config) + terminology, term_list = get_terminology_template(session, keywords or question, principal.oid, ds.id, + config=config) + training, example_list = get_training_template(session, question, principal.oid, ds.id, config=config) + return { + "datasource_id": str(ds.id), + "engine": _engine_with_version(resolved), + "schema": schema, + "tables": tables, + "terminologies": term_list, + "sql_examples": example_list, + "custom_prompt": _custom_prompt(session, principal, ds.id), + "rules": SQL_WRITING_RULES, + "more_tables_hint": "If a needed table is missing, call agent_list_tables and agent_describe_tables.", + } + + +# --- execute ----------------------------------------------------------------- + +def table_access_for(session: Session, principal: AgentPrincipal, ds: CoreDatasource) -> dict[str, TableAccess]: + """Tables the caller may read, with visible columns and row filters.""" + objs = get_table_obj_by_ds(session=session, current_user=principal.user, ds=ds) + if not objs: + return {} + normal = is_normal_user(principal.user) + filters = {} + if normal: + names = [obj.table.table_name for obj in objs] + for item in get_row_permission_filters(session=session, current_user=principal.user, ds=ds, tables=names): + filters[item["table"].lower()] = item["filter"] + + from apps.datasource.models.datasource import CoreField + table_ids = [obj.table.id for obj in objs] + checked = session.exec(select(CoreField.table_id, CoreField.field_name) + .where(CoreField.table_id.in_(table_ids), CoreField.checked == True)).all() # noqa: E712 + all_columns: dict[int, set[str]] = {} + for table_id, field_name in checked: + all_columns.setdefault(table_id, set()).add(field_name.lower()) + + access = {} + for obj in objs: + name = obj.table.table_name.lower() + visible = frozenset(f.field_name.lower() for f in (obj.fields or [])) + known = frozenset(all_columns.get(obj.table.id, set())) + restricted = normal and visible != known + access[name] = TableAccess( + name=name, + visible_columns=visible if restricted else None, + all_columns=known, + row_filter=filters.get(name), + schema=(obj.schema or "").lower() if ds.type not in ("mysql", "doris", "starrocks", "hive") else "", + ) + return access + + +def _chat_for(session: Session, principal: AgentPrincipal, ds: CoreDatasource, chat_id: Any, + question: str) -> int: + if chat_id: + try: + chat = session.get(Chat, int(chat_id)) + except (TypeError, ValueError): + chat = None + if chat is None or chat.create_by != principal.user.id or chat.oid != principal.oid \ + or (chat.datasource and int(chat.datasource) != ds.id): + raise ToolError("Chat not found or not accessible", code="not_found") + return chat.id + info = create_chat(session, principal.user, + CreateChat(origin=CHAT_ORIGIN_MCP, datasource=ds.id, question=question[:200]), + require_datasource=False) + return info.id + + +def execute_sql(session: Session, principal: AgentPrincipal, datasource_id: Any, sql: str, question: str, + chat_id: Any = None, max_rows: int | None = None) -> dict: + sql = (sql or "").strip() + question = (question or "").strip() + if not sql: + raise ToolError("sql must not be empty", code="invalid_argument") + if not question: + raise ToolError("question must not be empty", code="invalid_argument") + limit = max(1, min(int(max_rows or DEFAULT_MAX_ROWS), MAX_ROWS_LIMIT)) + + resolved = resolve_datasource(session, principal, datasource_id) + ds = resolved.ds + tables = table_access_for(session, principal, ds) + try: + ctx = build_context(sql.rstrip(";").strip(), ds.type, tables=tables, max_rows=limit, strict=True) + final_sql = get_pipeline(AGENT).run(ctx) + except SqlRejected as e: + raise ToolError(e.reason, code=e.code) + + if not check_connection(trans=None, ds=ds): + raise ToolError("Datasource connection failed", code="connection_failed") + + chat_id = _chat_for(session, principal, ds, chat_id, question) + record = save_question(session, principal.user, ChatQuestion(chat_id=chat_id, question=question)) + # The record keeps the agent's SQL; the permission-rewritten SQL is not + # stored or returned so row rules are not disclosed. + save_sql(session, record.id, sql) + try: + result = exec_sql(ds=ds, sql=final_sql, origin_column=False, + timeout_seconds=STATEMENT_TIMEOUT_SECONDS, max_rows=limit + 1) + except Exception as e: + message = str(e).splitlines()[0][:500] if str(e) else type(e).__name__ + save_error_message(session, record.id, orjson.dumps({"message": message, "type": "exec-sql-err"}).decode()) + SQLBotLogUtil.warning(f"[sqlbot-mcp] execute failed record={record.id}: {message}") + raise ToolError(f"SQL execution failed: {message}", code="execution_failed") + + rows = result.get("data") or [] + truncated = len(rows) > limit + rows = rows[:limit] + rows = DataFormat.normalize_qualified_sql_column_keys_in_object_array( + DataFormat.convert_large_numbers_in_object_array(rows)) + stored = {"fields": result.get("fields"), "fields_info": result.get("fields_info"), + "data": prepare_for_orjson(rows), "datasource": ds.id} + if truncated: + stored["limit"] = limit + save_sql_exec_data(session, record.id, orjson.dumps(stored).decode()) + finish_record(session, record.id) + return { + "chat_id": str(chat_id), + "record_id": str(record.id), + "fields": result.get("fields"), + "data": rows, + "row_count": len(rows), + "truncated": truncated, + "row_permission_applied": bool(ctx.notes.get("row_permission_applied")), + } diff --git a/backend/apps/mcp/sqlbot_mcp/tools/__init__.py b/backend/apps/mcp/sqlbot_mcp/tools/__init__.py new file mode 100644 index 00000000..c8cefebd --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/tools/__init__.py @@ -0,0 +1,17 @@ +"""Tool registry for the SQLBot MCP server. + +To add a tool: create a module with ``register(mcp, runtime)`` and append it +to ``TOOL_MODULES``. The server and the other tools stay unchanged. +""" + +from mcp.server.fastmcp import FastMCP + +from apps.mcp.sqlbot_mcp.tools import context, datasources, execute, tables +from apps.mcp.sqlbot_mcp.tools.base import ToolRuntime + +TOOL_MODULES = (datasources, context, tables, execute) + + +def register_tools(mcp: FastMCP, runtime: ToolRuntime, modules=TOOL_MODULES) -> None: + for module in modules: + module.register(mcp, runtime) diff --git a/backend/apps/mcp/sqlbot_mcp/tools/base.py b/backend/apps/mcp/sqlbot_mcp/tools/base.py new file mode 100644 index 00000000..7c9b00fe --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/tools/base.py @@ -0,0 +1,79 @@ +"""Shared plumbing for SQLBot MCP tools. + +A tool is a plain async function that receives the authenticated principal +implicitly and calls ``run_in_session`` for database work. Add a tool by +creating a module that defines ``register(mcp, runtime)`` and listing it in +``tools/__init__.py``. +""" + +import asyncio +import time +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, TypeVar + +from mcp.server.fastmcp.exceptions import ToolError as McpToolError +from sqlmodel import Session + +from apps.mcp.sqlbot_mcp.auth import AgentPrincipal, current_principal +from apps.mcp.sqlbot_mcp.ratelimit import RateLimited, RateLimiter +from apps.mcp.sqlbot_mcp.service import ToolError +from common.core.db import engine +from common.utils.utils import SQLBotLogUtil + +T = TypeVar("T") + + +class AuditSink: + """Records tool calls. Replace to ship audit events elsewhere.""" + + def record(self, *, principal: AgentPrincipal, tool: str, ok: bool, duration_ms: int, + detail: str | None = None, resource_id: str | None = None) -> None: # pragma: no cover + raise NotImplementedError + + +@dataclass +class ToolRuntime: + rate_limiter: RateLimiter + audit: AuditSink + session_factory: Callable[[], Session] = lambda: Session(engine) + + +async def run_tool(runtime: ToolRuntime, tool: str, work: Callable[[Session, AgentPrincipal], T], + *, audit: bool = False, resource_id: Callable[[T], str | None] = lambda _: None, + heavy: bool = False) -> T: + """Authenticate, rate-limit, run ``work`` in a thread, audit, map errors.""" + principal = current_principal() + try: + slot = runtime.rate_limiter.acquire(principal.credential_kind, principal.credential_id, heavy=heavy) + except RateLimited as e: + raise McpToolError(str(e)) + started = time.monotonic() + ok, detail, result = False, None, None + try: + result = await asyncio.to_thread(_in_session, runtime, principal, work) + ok = True + return result + except ToolError as e: + detail = f"{e.code}: {e.message}" + raise McpToolError(e.message) + except McpToolError: + raise + except Exception as e: + SQLBotLogUtil.exception(f"[sqlbot-mcp] tool {tool} failed") + detail = f"internal: {type(e).__name__}" + raise McpToolError("Internal error") + finally: + slot.release() + if audit or not ok: + try: + runtime.audit.record(principal=principal, tool=tool, ok=ok, + duration_ms=int((time.monotonic() - started) * 1000), detail=detail, + resource_id=resource_id(result) if ok else None) + except Exception: + SQLBotLogUtil.exception("[sqlbot-mcp] audit failed") + + +def _in_session(runtime: ToolRuntime, principal: AgentPrincipal, work: Callable[[Session, AgentPrincipal], Any]): + with runtime.session_factory() as session: + return work(session, principal) diff --git a/backend/apps/mcp/sqlbot_mcp/tools/context.py b/backend/apps/mcp/sqlbot_mcp/tools/context.py new file mode 100644 index 00000000..f82b5c79 --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/tools/context.py @@ -0,0 +1,31 @@ +from typing import Annotated, Any + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from apps.mcp.sqlbot_mcp import service +from apps.mcp.sqlbot_mcp.tools.base import ToolRuntime, run_tool + + +def register(mcp: FastMCP, runtime: ToolRuntime) -> None: + @mcp.tool(name="agent_get_query_context") + async def agent_get_query_context( + datasource_id: Annotated[str, Field(description="Datasource id from agent_list_datasources")], + question: Annotated[str, Field(description="The user's question in natural language")], + keywords: Annotated[str | None, Field( + description="Optional comma-separated business keywords you extracted from the question; " + "improves table and terminology matching")] = None, + ) -> dict[str, Any]: + """Get everything needed to write SQL for a question. + + Returns the database engine and version, the schema of the most relevant + tables (only columns you may read), table relations, matching business + terminology, similar question/SQL examples, custom instructions, and + SQL writing rules. Call this before agent_execute_sql for every new + question. No model is called; this is a lookup. + """ + return await run_tool( + runtime, "agent_get_query_context", + lambda session, principal: service.get_query_context(session, principal, datasource_id, question, + keywords), + ) diff --git a/backend/apps/mcp/sqlbot_mcp/tools/datasources.py b/backend/apps/mcp/sqlbot_mcp/tools/datasources.py new file mode 100644 index 00000000..1e936e8f --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/tools/datasources.py @@ -0,0 +1,18 @@ +from typing import Any + +from mcp.server.fastmcp import FastMCP + +from apps.mcp.sqlbot_mcp import service +from apps.mcp.sqlbot_mcp.tools.base import ToolRuntime, run_tool + + +def register(mcp: FastMCP, runtime: ToolRuntime) -> None: + @mcp.tool(name="agent_list_datasources") + async def agent_list_datasources() -> list[dict[str, Any]]: + """List the datasources you can query in SQLBot. + + Returns id, name, type, engine and description for each datasource. + Use the id with the other agent_* tools. + """ + return await run_tool(runtime, "agent_list_datasources", + lambda session, principal: service.list_datasources(session, principal)) diff --git a/backend/apps/mcp/sqlbot_mcp/tools/execute.py b/backend/apps/mcp/sqlbot_mcp/tools/execute.py new file mode 100644 index 00000000..257562fe --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/tools/execute.py @@ -0,0 +1,35 @@ +from typing import Annotated, Any + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from apps.mcp.sqlbot_mcp import service +from apps.mcp.sqlbot_mcp.tools.base import ToolRuntime, run_tool + + +def register(mcp: FastMCP, runtime: ToolRuntime) -> None: + @mcp.tool(name="agent_execute_sql") + async def agent_execute_sql( + datasource_id: Annotated[str, Field(description="Datasource id from agent_list_datasources")], + sql: Annotated[str, Field(description="One read-only SELECT statement in the datasource's dialect")], + question: Annotated[str, Field(description="The question this SQL answers; stored in SQLBot history")], + chat_id: Annotated[str | None, Field( + description="chat_id from a previous agent_execute_sql call to continue the same conversation")] = None, + max_rows: Annotated[int | None, Field( + description=f"Maximum rows to return (default {service.DEFAULT_MAX_ROWS}, " + f"at most {service.MAX_ROWS_LIMIT})", ge=1, le=service.MAX_ROWS_LIMIT)] = None, + ) -> dict[str, Any]: + """Validate and run one read-only SQL query, and save it to SQLBot history. + + SQLBot rejects SQL that writes, locks, calls unsafe functions, or reads + tables/columns you may not access, and explains why; fix the SQL and + retry. Row permissions and a row cap are applied automatically. + Prefer aggregated queries over large detail queries. Returns columns, + rows, `truncated`, and `chat_id`/`record_id`. + """ + return await run_tool( + runtime, "agent_execute_sql", + lambda session, principal: service.execute_sql(session, principal, datasource_id, sql, question, + chat_id, max_rows), + audit=True, heavy=True, resource_id=lambda result: result.get("record_id") if result else None, + ) diff --git a/backend/apps/mcp/sqlbot_mcp/tools/tables.py b/backend/apps/mcp/sqlbot_mcp/tools/tables.py new file mode 100644 index 00000000..90b5d862 --- /dev/null +++ b/backend/apps/mcp/sqlbot_mcp/tools/tables.py @@ -0,0 +1,41 @@ +from typing import Annotated, Any + +from mcp.server.fastmcp import FastMCP +from pydantic import Field + +from apps.mcp.sqlbot_mcp import service +from apps.mcp.sqlbot_mcp.tools.base import ToolRuntime, run_tool + + +def register(mcp: FastMCP, runtime: ToolRuntime) -> None: + @mcp.tool(name="agent_list_tables") + async def agent_list_tables( + datasource_id: Annotated[str, Field(description="Datasource id from agent_list_datasources")], + keyword: Annotated[str | None, Field( + description="Optional substring to filter table names and comments")] = None, + ) -> dict[str, Any]: + """List tables you can read in a datasource (names and comments only). + + Use this when agent_get_query_context did not include a table you need, + then call agent_describe_tables for the tables you pick. + """ + return await run_tool( + runtime, "agent_list_tables", + lambda session, principal: service.list_tables(session, principal, datasource_id, keyword), + ) + + @mcp.tool(name="agent_describe_tables") + async def agent_describe_tables( + datasource_id: Annotated[str, Field(description="Datasource id from agent_list_datasources")], + table_names: Annotated[list[str], Field(description="Table names to describe", min_length=1, + max_length=20)], + ) -> dict[str, Any]: + """Get columns and relations for specific tables. + + Only columns you may read are returned. Tables that do not exist or are + not accessible are listed in `not_accessible`. + """ + return await run_tool( + runtime, "agent_describe_tables", + lambda session, principal: service.describe_tables(session, principal, datasource_id, table_names), + ) diff --git a/backend/apps/system/api/agent_token.py b/backend/apps/system/api/agent_token.py new file mode 100644 index 00000000..758de3ab --- /dev/null +++ b/backend/apps/system/api/agent_token.py @@ -0,0 +1,105 @@ +"""Self-service management of SQLBot MCP agent tokens. + +Every route acts on the caller's own tokens only. The plain token is returned +once, by ``POST``; listings expose the prefix only. +""" + + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field, field_serializer + +from apps.system.crud import agent_token_manage as tokens +from apps.system.models.system_model import AgentTokenModel +from common.audit.models.log_model import OperationModules, OperationType +from common.audit.schemas.logger_decorator import LogConfig, system_log +from common.core.deps import CurrentUser, SessionDep, Trans + +router = APIRouter(tags=["system_agent_token"], prefix="/system/agent-token", include_in_schema=False) + + +class AgentTokenCreate(BaseModel): + name: str = Field(min_length=1, max_length=64) + # ``None`` follows the owner's datasource permissions. + datasource_ids: list[int] | None = None + + +class AgentTokenStatus(BaseModel): + id: int + status: bool + + +class AgentTokenItem(BaseModel): + id: int + name: str + token_prefix: str + status: bool + oid: int + datasource_ids: list[str] | None = None + create_time: int + last_used_time: int | None = None + last_used_ip: str | None = None + + @field_serializer("id", "oid") + def big_int_to_str(self, v: int) -> str: + return str(v) + + @classmethod + def of(cls, model: AgentTokenModel) -> "AgentTokenItem": + scope = tokens.load_ds_ids(model.allowed_ds_ids) + return cls( + id=model.id, name=model.name, token_prefix=f"{tokens.TOKEN_SCHEME}{model.token_prefix}", + status=model.status, oid=model.oid, + datasource_ids=None if scope is None else sorted(str(i) for i in scope), + create_time=model.create_time, last_used_time=model.last_used_time, last_used_ip=model.last_used_ip, + ) + + +class AgentTokenCreated(AgentTokenItem): + # Shown once; the server keeps only its digest. + token: str + + +def _raise(trans: Trans, error: tokens.AgentTokenError): + status = 404 if error.key == "not_found" else 400 + raise HTTPException(status_code=status, detail=trans(f"i18n_agent_token.{error.key}", **error.params)) + + +@router.get("") +async def list_agent_tokens(session: SessionDep, current_user: CurrentUser) -> list[AgentTokenItem]: + return [AgentTokenItem.of(m) for m in tokens.list_tokens(session, current_user.id)] + + +@router.post("") +@system_log(LogConfig(operation_type=OperationType.CREATE, module=OperationModules.AGENT_TOKEN, + result_id_expr="id", save_on_success_only=True)) +async def create_agent_token(session: SessionDep, current_user: CurrentUser, trans: Trans, + body: AgentTokenCreate) -> AgentTokenCreated: + try: + issued = tokens.create_token(session, uid=current_user.id, oid=current_user.oid, + name=body.name, ds_ids=body.datasource_ids) + except tokens.AgentTokenError as e: + _raise(trans, e) + item = AgentTokenItem.of(issued.model) + return AgentTokenCreated(**item.model_dump(), token=issued.plain) + + +@router.put("/status") +@system_log(LogConfig(operation_type=OperationType.UPDATE_STATUS, module=OperationModules.AGENT_TOKEN, + resource_id_expr="body.id")) +async def update_agent_token_status(session: SessionDep, current_user: CurrentUser, trans: Trans, + body: AgentTokenStatus) -> AgentTokenItem: + try: + model = tokens.set_token_status(session, uid=current_user.id, token_id=body.id, status=body.status) + except tokens.AgentTokenError as e: + _raise(trans, e) + return AgentTokenItem.of(model) + + +@router.delete("/{token_id}") +@system_log(LogConfig(operation_type=OperationType.DELETE, module=OperationModules.AGENT_TOKEN, + resource_id_expr="token_id")) +async def delete_agent_token(session: SessionDep, current_user: CurrentUser, trans: Trans, token_id: int): + try: + tokens.delete_token(session, uid=current_user.id, token_id=token_id) + except tokens.AgentTokenError as e: + _raise(trans, e) diff --git a/backend/apps/system/crud/agent_token_manage.py b/backend/apps/system/crud/agent_token_manage.py new file mode 100644 index 00000000..8013790c --- /dev/null +++ b/backend/apps/system/crud/agent_token_manage.py @@ -0,0 +1,208 @@ +"""Agent access tokens for the SQLBot MCP server. + +Format: ``sk-sqlbot--``. ``prefix`` (8 url-safe chars) locates +the row; only ``sha256(token)`` is stored and compared in constant time. +Plain tokens exist only in the create response. +""" + +import hashlib +import hmac +import json +import secrets +import string +import time +from dataclasses import dataclass + +from sqlmodel import Session, func, select + +from apps.datasource.models.datasource import CoreDatasource +from apps.system.models.system_model import AgentTokenModel, UserWsModel +from apps.system.models.user import UserModel +from common.utils.time import get_timestamp + +TOKEN_SCHEME = "sk-sqlbot-" +PREFIX_LENGTH = 8 +MAX_TOKENS_PER_USER = 5 +# ``last_used_*`` is refreshed at most this often per token. +LAST_USED_WRITE_INTERVAL_MS = 60 * 1000 + +_PREFIX_ALPHABET = string.ascii_lowercase + string.digits + + +class AgentTokenError(ValueError): + """Business rule violation; ``key`` is an i18n key under ``i18n_agent_token``.""" + + def __init__(self, key: str, **params): + super().__init__(key) + self.key = key + self.params = params + + +@dataclass(frozen=True) +class IssuedToken: + model: AgentTokenModel + plain: str + + +@dataclass(frozen=True) +class VerifiedToken: + """What a valid token resolves to; consumers never see the digest.""" + + token_id: int + name: str + uid: int + oid: int + allowed_ds_ids: frozenset[int] | None + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def generate_token() -> tuple[str, str]: + prefix = "".join(secrets.choice(_PREFIX_ALPHABET) for _ in range(PREFIX_LENGTH)) + return prefix, f"{TOKEN_SCHEME}{prefix}-{secrets.token_urlsafe(32)}" + + +def parse_prefix(token: str) -> str | None: + if not token or not token.startswith(TOKEN_SCHEME): + return None + rest = token[len(TOKEN_SCHEME):] + prefix, sep, secret = rest.partition("-") + if not sep or len(prefix) != PREFIX_LENGTH or not secret: + return None + if any(c not in _PREFIX_ALPHABET for c in prefix): + return None + return prefix + + +def dump_ds_ids(ds_ids: list[int] | None) -> str | None: + if ds_ids is None: + return None + return json.dumps(sorted({str(int(i)) for i in ds_ids})) + + +def load_ds_ids(raw: str | None) -> frozenset[int] | None: + if raw is None: + return None + try: + return frozenset(int(i) for i in json.loads(raw)) + except (ValueError, TypeError): + # A corrupt scope must never widen access: allow nothing. + return frozenset() + + +def _validate_scope(session: Session, oid: int, ds_ids: list[int] | None) -> None: + if ds_ids is None: + return + if not ds_ids: + raise AgentTokenError("datasource_required") + wanted = {int(i) for i in ds_ids} + found = set(session.exec( + select(CoreDatasource.id).where(CoreDatasource.id.in_(wanted), CoreDatasource.oid == oid) + ).all()) + if found != wanted: + raise AgentTokenError("datasource_invalid") + + +def list_tokens(session: Session, uid: int) -> list[AgentTokenModel]: + stmt = select(AgentTokenModel).where(AgentTokenModel.uid == uid).order_by(AgentTokenModel.create_time.desc()) + return list(session.exec(stmt).all()) + + +def create_token(session: Session, *, uid: int, oid: int, name: str, + ds_ids: list[int] | None = None) -> IssuedToken: + name = (name or "").strip() + if not name or len(name) > 64: + raise AgentTokenError("name_invalid") + if not oid: + raise AgentTokenError("workspace_required") + count = session.exec(select(func.count()).select_from(AgentTokenModel) + .where(AgentTokenModel.uid == uid)).one() + if count >= MAX_TOKENS_PER_USER: + raise AgentTokenError("limit", limit=MAX_TOKENS_PER_USER) + _validate_scope(session, oid, ds_ids) + + for _ in range(5): + prefix, plain = generate_token() + exists = session.exec(select(AgentTokenModel.id).where(AgentTokenModel.token_prefix == prefix)).first() + if not exists: + break + else: # pragma: no cover - 36^8 prefixes make this practically unreachable + raise AgentTokenError("generate_failed") + + model = AgentTokenModel( + name=name, uid=uid, oid=oid, token_prefix=prefix, token_hash=hash_token(plain), + status=True, allowed_ds_ids=dump_ds_ids(ds_ids), create_time=get_timestamp(), + ) + session.add(model) + session.commit() + session.refresh(model) + return IssuedToken(model=model, plain=plain) + + +def _owned(session: Session, uid: int, token_id: int) -> AgentTokenModel: + model = session.get(AgentTokenModel, token_id) + # Same error for "missing" and "someone else's" so ids cannot be probed. + if not model or model.uid != uid: + raise AgentTokenError("not_found") + return model + + +def set_token_status(session: Session, *, uid: int, token_id: int, status: bool) -> AgentTokenModel: + model = _owned(session, uid, token_id) + if model.status != status: + model.status = status + session.add(model) + session.commit() + session.refresh(model) + return model + + +def delete_token(session: Session, *, uid: int, token_id: int) -> AgentTokenModel: + model = _owned(session, uid, token_id) + snapshot = AgentTokenModel.model_validate(model.model_dump()) + session.delete(model) + session.commit() + return snapshot + + +def verify_token(session: Session, token: str) -> VerifiedToken | None: + """Return the token's identity, or ``None`` for any invalid/disabled token.""" + prefix = parse_prefix(token) + if prefix is None: + return None + model = session.exec(select(AgentTokenModel).where(AgentTokenModel.token_prefix == prefix)).first() + if model is None: + # Compare anyway so timing does not reveal whether the prefix exists. + hmac.compare_digest(hash_token(token), "0" * 64) + return None + if not hmac.compare_digest(hash_token(token), model.token_hash): + return None + if not model.status: + return None + user = session.get(UserModel, model.uid) + if user is None or user.status != 1: + return None + if model.uid != 1: # the built-in admin may use every workspace + member = session.exec(select(UserWsModel.id).where( + UserWsModel.uid == model.uid, UserWsModel.oid == model.oid)).first() + if member is None: + return None + return VerifiedToken(token_id=model.id, name=model.name, uid=model.uid, oid=model.oid, + allowed_ds_ids=load_ds_ids(model.allowed_ds_ids)) + + +def touch_token(session: Session, token_id: int, ip: str | None, *, now_ms: int | None = None) -> None: + """Record last use, throttled so every tool call does not write a row.""" + now_ms = now_ms if now_ms is not None else int(time.time() * 1000) + model = session.get(AgentTokenModel, token_id) + if model is None: + return + if model.last_used_time and now_ms - model.last_used_time < LAST_USED_WRITE_INTERVAL_MS \ + and model.last_used_ip == ip: + return + model.last_used_time = now_ms + model.last_used_ip = (ip or "")[:64] or None + session.add(model) + session.commit() diff --git a/backend/apps/system/models/system_model.py b/backend/apps/system/models/system_model.py index cf23e8d1..27e49dde 100644 --- a/backend/apps/system/models/system_model.py +++ b/backend/apps/system/models/system_model.py @@ -109,3 +109,25 @@ class ApiKeyBaseModel(SQLModel): class ApiKeyModel(SnowflakeBase, ApiKeyBaseModel, table=True): __tablename__ = "sys_apikey" + + +class AgentTokenModel(SnowflakeBase, table=True): + """Bearer token an external agent uses to call the SQLBot MCP server. + + The token acts as its owner (``uid``) inside one workspace (``oid``). Only a + SHA-256 digest of the secret is stored; ``token_prefix`` locates the row. + ``allowed_ds_ids`` optionally narrows the owner's datasources (JSON list of + ids as strings); ``None`` follows the owner's permissions. + """ + + __tablename__ = "sys_agent_token" + name: str = Field(max_length=255, nullable=False) + uid: int = Field(nullable=False, sa_type=BigInteger(), index=True) + oid: int = Field(nullable=False, sa_type=BigInteger()) + token_prefix: str = Field(max_length=32, nullable=False, unique=True, index=True) + token_hash: str = Field(max_length=128, nullable=False) + status: bool = Field(default=True, nullable=False) + allowed_ds_ids: Optional[str] = Field(default=None, sa_type=Text(), nullable=True) + create_time: int = Field(default=0, sa_type=BigInteger()) + last_used_time: Optional[int] = Field(default=None, sa_type=BigInteger(), nullable=True) + last_used_ip: Optional[str] = Field(default=None, max_length=64, nullable=True) diff --git a/backend/common/audit/models/log_model.py b/backend/common/audit/models/log_model.py index 8ecb2773..592e5af6 100644 --- a/backend/common/audit/models/log_model.py +++ b/backend/common/audit/models/log_model.py @@ -24,6 +24,8 @@ class OperationModules(str, Enum): THEME = "theme" # 外观配置 PARAMS_SETTING = "params_setting" # 参数配置 API_KEY = "api_key" # api key + AGENT_TOKEN = "agent_token" # sqlbot-mcp 访问令牌 + AGENT_MCP = "agent_mcp" # sqlbot-mcp 工具调用 LOG_SETTING = "log_setting" # api key SETTING = "setting" # 设置 SYSTEM_MANAGEMENT = "system_management" # 系统管理 @@ -50,6 +52,7 @@ class OperationType(str, Enum): CREATE_OR_UPDATE = "create_or_update" ANALYSIS = "analysis" PREDICTION = "prediction" + EXECUTE_SQL = "execute_sql" class SystemLogsResource(SQLModel, table=True): __tablename__ = "sys_logs_resource" diff --git a/backend/locales/en.json b/backend/locales/en.json index c57c57b0..ffdcdf8c 100644 --- a/backend/locales/en.json +++ b/backend/locales/en.json @@ -206,7 +206,10 @@ "reset_pwd": "Reset Password", "update_pwd": "Update Password", "update_status": "Update Status", - "update_table_relation": "Change Table Relation" + "update_table_relation": "Change Table Relation", + "agent_token": "Agent token", + "agent_mcp": "Agent query (sqlbot-mcp)", + "execute_sql": "Execute SQL" }, "i18n_table_not_exist": "Table not exist", "i18n_variable": { @@ -217,5 +220,14 @@ }, "i18n_parameter": { "invalid_embedding_parameter": "Invalid embedding parameter: {key}. Similarities and weights must be finite numbers from 0 to 1; candidate counts must be positive integers." + }, + "i18n_agent_token": { + "name_invalid": "Token name is required and must be at most 64 characters", + "workspace_required": "This account is not associated with a workspace", + "limit": "You can create at most {limit} agent tokens", + "datasource_required": "Select at least one datasource, or do not restrict datasources", + "datasource_invalid": "A selected datasource does not exist or is not in the current workspace", + "not_found": "Token not found", + "generate_failed": "Failed to generate a token, please retry" } } diff --git a/backend/locales/ko-KR.json b/backend/locales/ko-KR.json index e66ee2e3..0918fd3f 100644 --- a/backend/locales/ko-KR.json +++ b/backend/locales/ko-KR.json @@ -206,7 +206,10 @@ "reset_pwd": "비밀번호 재설정", "update_pwd": "비밀번호 업데이트", "update_status": "상태 업데이트", - "update_table_relation": "테이블 관계 변경" + "update_table_relation": "테이블 관계 변경", + "agent_token": "에이전트 토큰", + "agent_mcp": "에이전트 질의 (sqlbot-mcp)", + "execute_sql": "SQL 실행" }, "i18n_table_not_exist": "현재 테이블이 존재하지 않습니다", "i18n_variable": { @@ -217,5 +220,14 @@ }, "i18n_parameter": { "invalid_embedding_parameter": "벡터 매개변수 {key}이(가) 유효하지 않습니다. 유사도와 가중치는 0~1 사이의 유한한 수, 후보 수는 양의 정수여야 합니다." + }, + "i18n_agent_token": { + "name_invalid": "토큰 이름은 필수이며 64자를 넘을 수 없습니다", + "workspace_required": "이 계정은 워크스페이스에 연결되어 있지 않습니다", + "limit": "에이전트 토큰은 최대 {limit}개까지 만들 수 있습니다", + "datasource_required": "데이터 소스를 하나 이상 선택하거나 제한하지 마세요", + "datasource_invalid": "선택한 데이터 소스가 없거나 현재 워크스페이스에 속하지 않습니다", + "not_found": "토큰을 찾을 수 없습니다", + "generate_failed": "토큰 생성에 실패했습니다. 다시 시도하세요" } } diff --git a/backend/locales/th-TH.json b/backend/locales/th-TH.json index b994f70b..f962e8d6 100644 --- a/backend/locales/th-TH.json +++ b/backend/locales/th-TH.json @@ -206,7 +206,10 @@ "reset_pwd": "รีเซ็ตรหัสผ่าน", "update_pwd": "อัปเดตรหัสผ่าน", "update_status": "อัปเดตสถานะ", - "update_table_relation": "เปลี่ยนแปลงความสัมพันธ์ระหว่างตาราง" + "update_table_relation": "เปลี่ยนแปลงความสัมพันธ์ระหว่างตาราง", + "agent_token": "โทเค็นเอเจนต์", + "agent_mcp": "การสืบค้นของเอเจนต์ (sqlbot-mcp)", + "execute_sql": "เรียกใช้ SQL" }, "i18n_table_not_exist": "ไม่พบตารางปัจจุบัน", "i18n_variable": { @@ -217,5 +220,14 @@ }, "i18n_parameter": { "invalid_embedding_parameter": "พารามิเตอร์เวกเตอร์ {key} ไม่ถูกต้อง ความคล้ายคลึงและน้ำหนักต้องเป็นจำนวนจำกัดระหว่าง 0 ถึง 1 และจำนวนรายการต้องเป็นจำนวนเต็มบวก" + }, + "i18n_agent_token": { + "name_invalid": "ต้องระบุชื่อโทเค็นและยาวไม่เกิน 64 ตัวอักษร", + "workspace_required": "บัญชีนี้ไม่ได้เชื่อมโยงกับพื้นที่ทำงาน", + "limit": "สร้างโทเค็นเอเจนต์ได้สูงสุด {limit} รายการ", + "datasource_required": "เลือกแหล่งข้อมูลอย่างน้อยหนึ่งรายการ หรือไม่จำกัดแหล่งข้อมูล", + "datasource_invalid": "แหล่งข้อมูลที่เลือกไม่มีอยู่หรือไม่อยู่ในพื้นที่ทำงานปัจจุบัน", + "not_found": "ไม่พบโทเค็น", + "generate_failed": "สร้างโทเค็นไม่สำเร็จ โปรดลองอีกครั้ง" } } diff --git a/backend/locales/zh-CN.json b/backend/locales/zh-CN.json index 7d5049de..9d0c4163 100644 --- a/backend/locales/zh-CN.json +++ b/backend/locales/zh-CN.json @@ -206,7 +206,10 @@ "reset_pwd": "重置密码", "update_pwd": "更新密码", "update_status": "更新状态", - "update_table_relation": "变更表关系" + "update_table_relation": "变更表关系", + "agent_token": "Agent 令牌", + "agent_mcp": "Agent 问数(sqlbot-mcp)", + "execute_sql": "执行 SQL" }, "i18n_table_not_exist": "当前表不存在", "i18n_variable": { @@ -217,5 +220,14 @@ }, "i18n_parameter": { "invalid_embedding_parameter": "向量参数 {key} 无效。相似度和权重必须是 0~1 的有限数值,候选数量必须是正整数。" + }, + "i18n_agent_token": { + "name_invalid": "令牌名称不能为空,且不超过 64 个字符", + "workspace_required": "当前账号未关联工作空间", + "limit": "最多只能创建 {limit} 个 Agent 令牌", + "datasource_required": "至少选择一个数据源,或不限制数据源", + "datasource_invalid": "所选数据源不存在或不属于当前工作空间", + "not_found": "令牌不存在", + "generate_failed": "令牌生成失败,请重试" } } diff --git a/backend/locales/zh-TW.json b/backend/locales/zh-TW.json index 83489f28..eabe6eef 100644 --- a/backend/locales/zh-TW.json +++ b/backend/locales/zh-TW.json @@ -206,7 +206,10 @@ "reset_pwd": "重置密碼", "update_pwd": "更新密碼", "update_status": "更新狀態", - "update_table_relation": "變更表關係" + "update_table_relation": "變更表關係", + "agent_token": "Agent 權杖", + "agent_mcp": "Agent 問數(sqlbot-mcp)", + "execute_sql": "執行 SQL" }, "i18n_table_not_exist": "當前表不存在", "i18n_variable": { @@ -217,5 +220,14 @@ }, "i18n_parameter": { "invalid_embedding_parameter": "向量參數 {key} 無效。相似度與權重必須是 0~1 的有限數值,候選數量必須是正整數。" + }, + "i18n_agent_token": { + "name_invalid": "權杖名稱不能為空,且不超過 64 個字元", + "workspace_required": "目前帳號未關聯工作空間", + "limit": "最多只能建立 {limit} 個 Agent 權杖", + "datasource_required": "至少選擇一個資料來源,或不限制資料來源", + "datasource_invalid": "所選資料來源不存在或不屬於目前工作空間", + "not_found": "權杖不存在", + "generate_failed": "權杖產生失敗,請重試" } } diff --git a/backend/main.py b/backend/main.py index 4e3408ed..08f8eb2d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -24,6 +24,7 @@ from apps.datasource.utils.utils import initialize_datasource_keys from common.core.db import engine from apps.api import api_router +from apps.mcp.sqlbot_mcp.server import DEFAULT_MOUNT_PATH as SQLBOT_MCP_PATH, build_sqlbot_mcp, mount_sqlbot_mcp from apps.swagger.i18n import PLACEHOLDER_PREFIX, tags_metadata, i18n_list from apps.swagger.i18n import get_translation, DEFAULT_LANG from apps.system.crud.aimodel_manage import async_model_info @@ -220,7 +221,14 @@ async def custom_swagger_ui(request: Request): ) -mcp_app = FastAPI() +@asynccontextmanager +async def mcp_lifespan(_app: FastAPI): + # sqlbot-mcp (Streamable HTTP) needs its session manager running. + async with sqlbot_mcp_lifespan(): + yield + + +mcp_app = FastAPI(lifespan=mcp_lifespan) mcp_app.add_middleware(McpClientIpForwardMiddleware) # mcp server, images path images_path = settings.MCP_IMAGE_PATH @@ -240,6 +248,11 @@ async def custom_swagger_ui(request: Request): mcp.mount(mcp_app) +# sqlbot-mcp: model-free tools for external agents, authenticated by agent +# tokens (Authorization: Bearer sk-sqlbot-...). Independent of the legacy /mcp. +sqlbot_mcp_server = build_sqlbot_mcp() +sqlbot_mcp_lifespan = mount_sqlbot_mcp(mcp_app, sqlbot_mcp_server, SQLBOT_MCP_PATH) + # Set all CORS enabled origins if settings.all_cors_origins: app.add_middleware( diff --git a/backend/tests/test_agent_token.py b/backend/tests/test_agent_token.py new file mode 100644 index 00000000..84e67ab6 --- /dev/null +++ b/backend/tests/test_agent_token.py @@ -0,0 +1,160 @@ +"""Offline regressions for sqlbot-mcp agent tokens (SQLite, no services).""" + +import json + +import pytest +import sqlbot_xpack # noqa: F401 # Initialize the published wheel before app imports. +from sqlalchemy import BigInteger, Column, MetaData, Text +from sqlalchemy.pool import StaticPool +from sqlmodel import Field, Session, SQLModel, create_engine, select + +from apps.system.crud import agent_token_manage as tokens +from apps.system.models.system_model import AgentTokenModel, UserWsModel + + +# Minimal stand-ins on their own metadata: the real user and datasource models +# use PostgreSQL-only column types and must not be redefined. +class _FakeBase(SQLModel): + metadata = MetaData() + + +class FakeUser(_FakeBase, table=True): + __tablename__ = "sys_user" + id: int = Field(primary_key=True) + account: str = "u" + name: str = "u" + oid: int = 1 + status: int = 1 + + +class FakeDatasource(_FakeBase, table=True): + __tablename__ = "core_datasource" + id: int = Field(sa_column=Column(BigInteger, primary_key=True)) + oid: int = Field(sa_column=Column(BigInteger())) + name: str = Field(sa_column=Column(Text), default="ds") + + +@pytest.fixture +def session(monkeypatch): + engine = create_engine("sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool) + SQLModel.metadata.create_all(engine, tables=[AgentTokenModel.__table__, UserWsModel.__table__]) + _FakeBase.metadata.create_all(engine) + monkeypatch.setattr(tokens, "UserModel", FakeUser) + monkeypatch.setattr(tokens, "CoreDatasource", FakeDatasource) + with Session(engine) as s: + s.add(FakeUser(id=2, status=1)) + s.add(UserWsModel(uid=2, oid=10, weight=0)) + s.add(FakeDatasource(id=101, oid=10)) + s.add(FakeDatasource(id=202, oid=99)) + s.commit() + yield s + + +def issue(session, **kwargs): + defaults = {"uid": 2, "oid": 10, "name": "claude"} + defaults.update(kwargs) + return tokens.create_token(session, **defaults) + + +def test_token_format_and_storage(session): + issued = issue(session) + assert issued.plain.startswith("sk-sqlbot-") + assert tokens.parse_prefix(issued.plain) == issued.model.token_prefix + stored = session.exec(select(AgentTokenModel)).one() + # Only the digest is persisted. + assert stored.token_hash == tokens.hash_token(issued.plain) + assert issued.plain not in json.dumps(stored.model_dump(), default=str) + + +def test_verify_resolves_owner_workspace_and_scope(session): + issued = issue(session, ds_ids=[101]) + verified = tokens.verify_token(session, issued.plain) + assert (verified.uid, verified.oid, verified.allowed_ds_ids) == (2, 10, frozenset({101})) + + +@pytest.mark.parametrize("mutate", [ + lambda t: t[:-1] + ("A" if t[-1] != "A" else "B"), + lambda t: "sk-sqlbot-zzzzzzzz-" + t.split("-", 3)[-1], + lambda t: t.replace("sk-sqlbot-", "sk-other-"), + lambda t: "", + lambda t: "Bearer " + t, +]) +def test_tampered_tokens_are_rejected(session, mutate): + issued = issue(session) + assert tokens.verify_token(session, mutate(issued.plain)) is None + + +def test_disabled_or_deleted_tokens_are_rejected(session): + issued = issue(session) + tokens.set_token_status(session, uid=2, token_id=issued.model.id, status=False) + assert tokens.verify_token(session, issued.plain) is None + tokens.set_token_status(session, uid=2, token_id=issued.model.id, status=True) + assert tokens.verify_token(session, issued.plain) is not None + tokens.delete_token(session, uid=2, token_id=issued.model.id) + assert tokens.verify_token(session, issued.plain) is None + + +def test_owner_disabled_or_removed_from_workspace_invalidates_token(session): + issued = issue(session) + user = session.get(FakeUser, 2) + user.status = 0 + session.add(user) + session.commit() + assert tokens.verify_token(session, issued.plain) is None + user.status = 1 + session.add(user) + for ws in session.exec(select(UserWsModel)).all(): + session.delete(ws) + session.commit() + assert tokens.verify_token(session, issued.plain) is None + + +def test_users_cannot_manage_other_users_tokens(session): + issued = issue(session) + for action in ( + lambda: tokens.set_token_status(session, uid=3, token_id=issued.model.id, status=False), + lambda: tokens.delete_token(session, uid=3, token_id=issued.model.id), + ): + with pytest.raises(tokens.AgentTokenError) as info: + action() + assert info.value.key == "not_found" + assert tokens.list_tokens(session, 3) == [] + assert tokens.verify_token(session, issued.plain) is not None + + +@pytest.mark.parametrize("ds_ids,key", [([202], "datasource_invalid"), ([999], "datasource_invalid"), + ([], "datasource_required")]) +def test_scope_must_stay_inside_the_workspace(session, ds_ids, key): + with pytest.raises(tokens.AgentTokenError) as info: + issue(session, ds_ids=ds_ids) + assert info.value.key == key + + +def test_per_user_limit(session): + for i in range(tokens.MAX_TOKENS_PER_USER): + issue(session, name=f"t{i}") + with pytest.raises(tokens.AgentTokenError) as info: + issue(session, name="one-too-many") + assert info.value.key == "limit" + + +@pytest.mark.parametrize("name", ["", " ", "x" * 65]) +def test_name_is_validated(session, name): + with pytest.raises(tokens.AgentTokenError): + issue(session, name=name) + + +def test_corrupt_scope_never_widens_access(): + assert tokens.load_ds_ids("not json") == frozenset() + assert tokens.load_ds_ids(None) is None + + +def test_last_used_is_throttled(session): + issued = issue(session) + tokens.touch_token(session, issued.model.id, "1.1.1.1", now_ms=1_000_000) + tokens.touch_token(session, issued.model.id, "1.1.1.1", now_ms=1_010_000) + model = session.get(AgentTokenModel, issued.model.id) + assert model.last_used_time == 1_000_000 + tokens.touch_token(session, issued.model.id, "1.1.1.1", now_ms=1_000_000 + tokens.LAST_USED_WRITE_INTERVAL_MS) + session.refresh(model) + assert model.last_used_time == 1_000_000 + tokens.LAST_USED_WRITE_INTERVAL_MS diff --git a/backend/tests/test_sqlbot_mcp.py b/backend/tests/test_sqlbot_mcp.py new file mode 100644 index 00000000..bb6a25a1 --- /dev/null +++ b/backend/tests/test_sqlbot_mcp.py @@ -0,0 +1,179 @@ +"""Offline regressions for the sqlbot-mcp server wiring. + +A real FastMCP server is mounted on a fresh Starlette app and driven by the +official MCP Streamable HTTP client over an in-process ASGI transport. The +authenticator, audit sink and services are test doubles; no database, model +or network is used. +""" + +import contextlib +import json +from types import SimpleNamespace + +import httpx +import pytest +import sqlbot_xpack # noqa: F401 # Initialize the published wheel before app imports. +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client +from starlette.applications import Starlette + +from apps.mcp.sqlbot_mcp import service +from apps.mcp.sqlbot_mcp.auth import ( + AgentPrincipal, + AuthenticationFailed, + current_principal, +) +from apps.mcp.sqlbot_mcp.ratelimit import InMemoryRateLimiter +from apps.mcp.sqlbot_mcp.server import SERVER_NAME, build_sqlbot_mcp, mount_sqlbot_mcp +from apps.mcp.sqlbot_mcp.tools import base as tool_base + +GOOD = "sk-sqlbot-goodgood-secret" + + +class FakeAuthenticator: + async def authenticate(self, request): + if request.headers.get("authorization") != f"Bearer {GOOD}": + raise AuthenticationFailed("bad") + user = SimpleNamespace(id=2, oid=10, account="u", name="u") + return AgentPrincipal(user=user, credential_id="7", credential_kind="agent_token", + allowed_ds_ids=frozenset({101}), client_ip="1.2.3.4") + + +class RecordingAudit(tool_base.AuditSink): + def __init__(self): + self.events = [] + + def record(self, **kwargs): + self.events.append(kwargs) + + +@pytest.fixture +def harness(monkeypatch): + audit = RecordingAudit() + seen = {} + + def list_datasources(_session, principal): + seen["principal"] = principal + return [{"id": "101", "name": "sales"}] + + def execute_sql(_session, _principal, _datasource_id, sql, *_rest): + if "forbidden" in sql: + raise service.ToolError("SQL references tables that are not accessible: forbidden", code="table_access") + if "boom" in sql: + raise RuntimeError("database password=secret leaked?") + return {"chat_id": "1", "record_id": "9", "fields": ["n"], "data": [{"n": 1}], "truncated": False} + + monkeypatch.setattr(service, "list_datasources", list_datasources) + monkeypatch.setattr(service, "execute_sql", execute_sql) + + server = build_sqlbot_mcp(authenticator=FakeAuthenticator(), audit=audit, + rate_limiter=InMemoryRateLimiter(per_minute=5, max_concurrent_heavy=1), + session_factory=lambda: contextlib.nullcontext(None)) + host = Starlette() + lifespan = mount_sqlbot_mcp(host, server, "/sqlbot-mcp") + return SimpleNamespace(app=host, lifespan=lifespan, audit=audit, seen=seen) + + +@contextlib.asynccontextmanager +async def client(harness, token=GOOD, path="/sqlbot-mcp"): + headers = {"Authorization": f"Bearer {token}"} if token else {} + + def factory(headers=None, timeout=None, auth=None): + return httpx.AsyncClient(transport=httpx.ASGITransport(app=harness.app), base_url="http://test", + headers=headers, timeout=timeout, auth=auth) + + async with harness.lifespan(): + async with streamablehttp_client(f"http://test{path}", headers=headers, + httpx_client_factory=factory) as (read, write, _): + async with ClientSession(read, write) as session: + yield session + + +def payload(result): + return json.loads(result.content[0].text) + + +@pytest.mark.anyio +@pytest.mark.parametrize("path", ["/sqlbot-mcp", "/sqlbot-mcp/"]) +async def test_tools_are_listed_under_the_server_name(harness, path): + async with client(harness, path=path) as session: + init = await session.initialize() + tools = await session.list_tools() + assert init.serverInfo.name == SERVER_NAME + assert {t.name for t in tools.tools} == { + "agent_list_datasources", "agent_get_query_context", "agent_list_tables", + "agent_describe_tables", "agent_execute_sql", + } + + +@pytest.mark.anyio +@pytest.mark.parametrize("headers", [{}, {"Authorization": "Bearer nope"}, {"Authorization": GOOD}]) +@pytest.mark.parametrize("path", ["/sqlbot-mcp", "/sqlbot-mcp/"]) +async def test_unauthenticated_requests_get_401_without_redirect(harness, headers, path): + body = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"} + async with harness.lifespan(): + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=harness.app), base_url="http://test") as c: + response = await c.post(path, json=body, headers={**headers, "Accept": "application/json, text/event-stream"}) + assert response.status_code == 401 + assert response.headers["www-authenticate"].startswith("Bearer") + + +@pytest.mark.anyio +async def test_tools_receive_the_authenticated_principal(harness): + async with client(harness) as session: + await session.initialize() + result = await session.call_tool("agent_list_datasources", {}) + assert not result.isError + assert (result.structuredContent or {}).get("result") == [{"id": "101", "name": "sales"}] + assert harness.seen["principal"].credential_id == "7" + + +@pytest.mark.anyio +async def test_rejections_are_returned_and_audited(harness): + async with client(harness) as session: + await session.initialize() + ok = await session.call_tool("agent_execute_sql", {"datasource_id": "101", "sql": "SELECT 1 FROM t", + "question": "q"}) + denied = await session.call_tool("agent_execute_sql", {"datasource_id": "101", + "sql": "SELECT * FROM forbidden", "question": "q"}) + assert not ok.isError + assert denied.isError and "not accessible" in denied.content[0].text + outcomes = [(e["tool"], e["ok"]) for e in harness.audit.events] + assert outcomes == [("agent_execute_sql", True), ("agent_execute_sql", False)] + assert harness.audit.events[0]["resource_id"] == "9" + assert "table_access" in harness.audit.events[1]["detail"] + + +@pytest.mark.anyio +async def test_internal_errors_do_not_leak_details(harness): + async with client(harness) as session: + await session.initialize() + result = await session.call_tool("agent_execute_sql", {"datasource_id": "101", "sql": "SELECT boom FROM t", + "question": "q"}) + assert result.isError + assert "secret" not in result.content[0].text + assert result.content[0].text.endswith("Internal error") + + +@pytest.mark.anyio +async def test_rate_limit_applies_per_credential(harness): + async with client(harness) as session: + await session.initialize() + results = [await session.call_tool("agent_list_datasources", {}) for _ in range(6)] + assert [r.isError for r in results] == [False] * 5 + [True] + assert "Rate limit" in results[-1].content[0].text + + +def test_tools_refuse_to_run_without_a_principal(): + with pytest.raises(PermissionError): + current_principal() + + +def test_heavy_calls_are_capped_and_released(): + limiter = InMemoryRateLimiter(per_minute=100, max_concurrent_heavy=1) + slot = limiter.acquire("agent_token", "7", heavy=True) + with pytest.raises(Exception, match="Too many queries"): + limiter.acquire("agent_token", "7", heavy=True) + limiter.acquire("agent_token", "8", heavy=True) # other credentials are independent + slot.release() + limiter.acquire("agent_token", "7", heavy=True) diff --git a/backend/tests/test_sqlbot_mcp_integration.py b/backend/tests/test_sqlbot_mcp_integration.py new file mode 100644 index 00000000..c848c5c8 --- /dev/null +++ b/backend/tests/test_sqlbot_mcp_integration.py @@ -0,0 +1,216 @@ +"""Opt-in end-to-end acceptance for sqlbot-mcp on a disposable PostgreSQL database. + +Run explicitly with SQLBOT_RUN_SQLBOT_MCP_INTEGRATION=1 and +SQLBOT_TEST_DATABASE_URL pointing at an authorized, *empty, disposable* +PostgreSQL database (with the pgvector extension available). The test runs the +real Alembic chain there, seeds its own rows and a ``hr`` schema used as the +queried datasource, and drops what it created. + +The production ``main:mcp_app`` runs in a child Uvicorn process with **no AI +model configured**; only the embedding model is replaced. Agent-token +authentication, datasource permissions, the SQL guard, real PostgreSQL +execution, chat records and audit logs are all real. +""" + +import asyncio +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client +from sqlalchemy import text +from sqlalchemy.engine import make_url +from sqlmodel import Session, create_engine, select + +pytestmark = pytest.mark.skipif( + os.environ.get("SQLBOT_RUN_SQLBOT_MCP_INTEGRATION") != "1", + reason="explicit disposable PostgreSQL integration opt-in required", +) + +BACKEND = Path(__file__).parents[1] + + +def serve_controlled_process(app_name, port): + """Child entry point: only the embedding model is replaced.""" + import uvicorn + from test_query_embedding import embedding_model + + from apps.ai_model.embedding import EmbeddingModelCache + + EmbeddingModelCache.get_model = staticmethod(lambda: embedding_model(limit=512)) + uvicorn.run("main:" + app_name, host="127.0.0.1", port=port, lifespan="on", log_level="warning") + + +@pytest.fixture +def environment(tmp_path): + url = make_url(os.environ["SQLBOT_TEST_DATABASE_URL"]) + env = dict(os.environ) + env.update({ + "POSTGRES_SERVER": url.host or "127.0.0.1", + "POSTGRES_PORT": str(url.port or 5432), + "POSTGRES_USER": url.username or "", + "POSTGRES_PASSWORD": url.password or "", + "POSTGRES_DB": url.database or "", + "MCP_IMAGE_PATH": str(tmp_path / "images"), + "EMBEDDING_ENABLED": "false", + "TABLE_EMBEDDING_ENABLED": "false", + "TABLE_EMBEDDING_KEYWORD_ENABLED": "false", + "PYTHONPATH": str(BACKEND / "tests") + os.pathsep + env.get("PYTHONPATH", ""), + }) + database = create_engine(url.render_as_string(hide_password=False)) + with database.connect() as connection: + existing = connection.execute(text( + "select count(*) from information_schema.tables where table_schema = 'public'")).scalar_one() + assert existing == 0, "SQLBOT_TEST_DATABASE_URL must point at an empty disposable database" + # Migrate the way the application does (``main.run_migrations`` in the + # lifespan): model modules are imported first, which some historical + # migrations rely on (e.g. 039 uses ``pgvector.sqlalchemy`` without + # importing it), so the bare ``alembic`` CLI is not equivalent. + migrate = subprocess.run([sys.executable, "-c", "import main; main.run_migrations()"], cwd=BACKEND, env=env, + capture_output=True, text=True, timeout=900) + assert migrate.returncode == 0, migrate.stdout[-2000:] + migrate.stderr[-4000:] + try: + yield {"url": url, "database": database, "env": env, "tmp_path": tmp_path} + finally: + database.dispose() + + +def _seed(env_info): + import sqlbot_xpack # noqa: F401 + + from apps.datasource.models.datasource import ( + CoreDatasource, + CoreField, + CoreTable, + DatasourceConf, + ) + from apps.datasource.utils.utils import aes_encrypt + from apps.system.crud import agent_token_manage as tokens + from apps.system.models.system_model import ( + AiModelDetail, + UserWsModel, + WorkspaceModel, + ) + from apps.system.models.user import UserModel + + url, database = env_info["url"], env_info["database"] + with database.begin() as connection: + connection.execute(text("CREATE SCHEMA hr")) + connection.execute(text("CREATE TABLE hr.emp (id int, name text, salary int, region text)")) + connection.execute(text("CREATE TABLE hr.secret (id int)")) + connection.execute(text( + "INSERT INTO hr.emp VALUES (1,'ann',100,'east'),(2,'bob',200,'west'),(3,'cid',300,'east')")) + with Session(database) as session: + assert session.exec(select(AiModelDetail)).first() is None, "no AI model must be configured" + session.add(WorkspaceModel(id=10, name="ws")) + session.add(UserModel(id=2, account="agent_user", name="agent", email="a@b.c", oid=10, status=1, + origin=0, language="zh-CN")) + session.add(UserWsModel(uid=2, oid=10, weight=0)) + # Store only the connection fields, like the datasource form does + # (``DatasourceConf``'s own ``sheets`` default does not round-trip). + conf = {"host": url.host, "port": url.port or 5432, "username": url.username, "password": url.password, + "database": url.database, "dbSchema": "hr", "timeout": 10} + DatasourceConf(**conf) + # aes_encrypt returns bytes; datasources store the base64 text. + configuration = aes_encrypt(json.dumps(conf)).decode() + hr = CoreDatasource(name="hr", description="hr data", type="pg", type_name="PostgreSQL", + configuration=configuration, oid=10, create_by=1, status="Success", + recommended_config=1) + other = CoreDatasource(name="other", description="", type="pg", type_name="PostgreSQL", + configuration=configuration, oid=99, create_by=1, status="Success", + recommended_config=1) + session.add(hr) + session.add(other) + session.flush() + emp = CoreTable(ds_id=hr.id, checked=True, table_name="emp", table_comment="employees", custom_comment="") + session.add(emp) + session.flush() + for index, (name, kind) in enumerate([("id", "int4"), ("name", "text"), ("salary", "int4"), + ("region", "text")]): + session.add(CoreField(ds_id=hr.id, table_id=emp.id, checked=True, field_name=name, field_type=kind, + field_comment="", custom_comment="", field_index=index)) + session.commit() + issued = tokens.create_token(session, uid=2, oid=10, name="it") + disabled = tokens.create_token(session, uid=2, oid=10, name="off") + tokens.set_token_status(session, uid=2, token_id=disabled.model.id, status=False) + return {"token": issued.plain, "disabled": disabled.plain, "token_id": issued.model.id, + "ds_id": hr.id, "other_ds_id": other.id} + + +async def _session_call(url, token, calls): + headers = {"Authorization": f"Bearer {token}"} + async with streamablehttp_client(url, headers=headers, timeout=30) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + return [await session.call_tool(name, args) for name, args in calls] + + +def _status(url, token): + import httpx + + headers = {"Accept": "application/json, text/event-stream"} + if token: + headers["Authorization"] = f"Bearer {token}" + body = {"jsonrpc": "2.0", "id": 1, "method": "tools/list"} + return httpx.post(url, json=body, headers=headers, timeout=10).status_code + + +def test_agent_queries_without_any_model(environment): + from test_mcp_parameters_integration import server + + from apps.chat.models.chat_model import Chat, ChatRecord + from apps.system.models.system_model import AgentTokenModel + from common.audit.models.log_model import SystemLog + + seed = _seed(environment) + ds = str(seed["ds_id"]) + with server("mcp_app", environment["env"], environment["tmp_path"], + provider_module="test_sqlbot_mcp_integration") as base: + url = base + "/sqlbot-mcp" + for bad in ("", "sk-sqlbot-zzzzzzzz-nope", seed["disabled"]): + assert _status(url, bad) == 401 + assert _status(base + "/api/v1/system/agent-token", seed["token"]) in (401, 404) + + results = asyncio.run(_session_call(url, seed["token"], [ + ("agent_list_datasources", {}), + ("agent_get_query_context", {"datasource_id": ds, "question": "average salary by region"}), + ("agent_execute_sql", {"datasource_id": ds, "question": "average salary by region", + "sql": "SELECT region, avg(salary) AS avg_salary FROM emp " + "GROUP BY region ORDER BY region"}), + ("agent_execute_sql", {"datasource_id": ds, "sql": "SELECT * FROM secret", "question": "q"}), + ("agent_execute_sql", {"datasource_id": ds, "question": "q", + "sql": "WITH x AS (DELETE FROM emp RETURNING *) SELECT * FROM x"}), + ("agent_execute_sql", {"datasource_id": str(seed["other_ds_id"]), "sql": "SELECT id FROM emp", + "question": "q"}), + ("agent_execute_sql", {"datasource_id": ds, "sql": "SELECT name FROM emp ORDER BY id", + "question": "names", "max_rows": 2}), + ])) + listed, context, ok, denied_table, denied_write, denied_ds, capped = results + + assert not listed.isError, listed + assert {d["name"] for d in listed.structuredContent["result"]} == {"hr"} + assert not context.isError, context + assert "emp" in context.structuredContent["schema"] + assert context.structuredContent["engine"].startswith("PostgreSQL") + assert not ok.isError, ok + assert [r["region"] for r in ok.structuredContent["data"]] == ["east", "west"] + assert denied_table.isError and "not accessible" in denied_table.content[0].text + assert denied_write.isError and "write operation" in denied_write.content[0].text + assert denied_ds.isError and "not found" in denied_ds.content[0].text.lower() + assert not capped.isError and capped.structuredContent["truncated"] is True + assert [r["name"] for r in capped.structuredContent["data"]] == ["ann", "bob"] + + with Session(environment["database"]) as session: + # The queried data is intact (the nested DELETE never ran). + assert session.exec(text("SELECT count(*) FROM hr.emp")).one()[0] == 3 + record = session.get(ChatRecord, int(ok.structuredContent["record_id"])) + chat = session.get(Chat, record.chat_id) + assert chat.origin == 1 and chat.create_by == 2 and chat.oid == 10 + assert record.finish and not record.error and "GROUP BY region" in record.sql + audits = session.exec(select(SystemLog).where(SystemLog.module == "agent_mcp")).all() + assert {a.operation_status for a in audits} == {"success", "failed"} + assert session.get(AgentTokenModel, seed["token_id"]).last_used_time is not None diff --git a/docs/agents/backend.md b/docs/agents/backend.md index 4cd1078a..29ba9fc2 100644 --- a/docs/agents/backend.md +++ b/docs/agents/backend.md @@ -15,6 +15,8 @@ - 项目同时存在 `crud` 和 `curd` 拼写;不要为统一命名制造无关重构。 - 新 router 注册到 `backend/apps/api.py`。 - 应用组装、中间件、MCP 和 MCP 图片静态挂载属于 `backend/main.py`;`init_fastapi_app` 在此触发 xpack 初始化,前端 dist 静态挂载与 `/xpack_static` 提取由 xpack 包内完成(见 `docs/agents/xpack.md`)。 +- MCP 服务进程(`main:mcp_app`,8001)有两个入口:`/mcp` 由 `fastapi_mcp` 把 `apps/mcp/mcp.py` 的 HTTP 接口暴露为工具(SSE);`/sqlbot-mcp` 是 `apps/mcp/sqlbot_mcp/` 用官方 `mcp` SDK 编写的无模型服务(Streamable HTTP,无状态)。`/sqlbot-mcp` 的扩展点:新工具在 `tools/` 新建模块并登记到 `TOOL_MODULES`;鉴权、限流、审计分别实现 `Authenticator`、`RateLimiter`、`AuditSink` 并传给 `build_sqlbot_mcp`;工具只依赖 `AgentPrincipal`,不读请求头。 +- SQL 校验与方言差异分别在 `apps/db/sql_guard/`、`apps/db/dialect/`,规则见 `docs/agents/security.md`。 - SQLModel 结构变更必须配套 Alembic 迁移;细节见 `docs/agents/migrations.md`。 - 用户可见后端消息使用 `backend/locales/`,不要硬编码新文案。 diff --git a/docs/agents/security.md b/docs/agents/security.md index 111f1bda..db48ac64 100644 --- a/docs/agents/security.md +++ b/docs/agents/security.md @@ -47,6 +47,29 @@ - 动态 SQL 生成后仍必须走同一解析、校验和保存路径。 - 数据源连接配置在服务端解密。完整配置、凭据和存储密钥不得进入日志或 API 响应。编辑详情的字段边界见「数据源配置存储」。 +### SQL 校验规则集(`backend/apps/db/sql_guard/`) + +- `check_sql_read` 使用 `basic` 规则集,所有执行入口共用;规则逐条为独立 `SqlCheck`,新增规则加入规则集,不要在调用方另写判断。 +- 数据源方言、危险函数、资源类函数和超时方式登记在 `backend/apps/db/dialect/` 的 `DialectPolicy`;新增数据源类型先注册策略,未注册类型一律拒绝。 +- 修改 `DialectPolicy.sqlglot_dialect` 会改变网页问数 SQL 的解析结果,需用各方言样例对照改动前后的 `check_sql_read` 结论。 +- 任何规则或改写抛出异常都按拒绝处理,不得降级为少做一步检查。 + +### 外部提交 SQL(`/sqlbot-mcp` 的 `agent_execute_sql`) + +SQL 由外部 Agent 编写,按不可信输入处理。使用 `agent` 规则集,并以 `strict=True` 按数据源精确方言解析和生成,顺序为: + +1. 校验数据源属于令牌的工作空间且在令牌的数据源范围内;越权与不存在返回同一错误; +2. `basic` 规则; +3. 仅单条语句;树中任意位置的写操作、锁子句、`SELECT ... INTO`、表提示、表函数一律拒绝; +4. 危险函数、身份类函数和资源类函数,同时检查具名节点、`Anonymous` 节点和 `包.函数` 形式; +5. 按作用域解析真实表(CTE 同名遮蔽按真实表处理),全部在可访问表内,系统库/其他 schema 拒绝;拒绝信息不列出可访问表; +6. 列权限:受限表上的 `*`、`t.*` 和任何位置引用的隐藏列均拒绝(包括 WHERE、JOIN、ORDER BY); +7. 行权限改写为 `(SELECT * FROM 表 WHERE 条件) AS 原别名`,覆盖所有出现位置,无法改写即拒绝;随后加行数上限;改写后的 SQL 再跑一遍可复检规则; +8. 执行时按方言设置语句超时,行数在改写和读取两层限制;会话记录只保存 Agent 提交的原 SQL,不保存、不返回改写后的 SQL; +9. 成功和拒绝都写入操作日志(`agent_mcp` 模块),不记录查询结果。 + +Agent 令牌只在 `/sqlbot-mcp` 子应用中校验,主应用 `/api/v1/*` 不接受该令牌;`/sqlbot-mcp` 也不接受网页登录 token 和 API Key。不要把 `/sqlbot-mcp` 加入 `whitelist.py`,也不要把新路由起成以 `/mcp` 开头的路径(`/mcp*` 在白名单内)。 + ## 认证、Host 与嵌入 - 修改 `backend/main.py` 的中间件顺序时,验证实际请求执行顺序(不能只看注册顺序);`HostValidationMiddleware` 必须在认证/业务使用 Host 前拒绝非法值,并回归预检与正常请求。 diff --git a/frontend/src/api/agentToken.ts b/frontend/src/api/agentToken.ts new file mode 100644 index 00000000..bf0af49a --- /dev/null +++ b/frontend/src/api/agentToken.ts @@ -0,0 +1,34 @@ +import { request } from '@/utils/request' + +export interface AgentToken { + id: string + name: string + // Display prefix only, e.g. "sk-sqlbot-ab12cd34"; the secret is never listed. + token_prefix: string + status: boolean + oid: string + // null: follows the owner's datasource permissions. + datasource_ids: string[] | null + create_time: number + last_used_time: number | null + last_used_ip: string | null +} + +export interface AgentTokenCreated extends AgentToken { + // Full token, returned once by create. + token: string +} + +export interface AgentTokenCreatePayload { + name: string + datasource_ids: string[] | null +} + +export const agentTokenApi = { + list: (): Promise => request.get('/system/agent-token'), + create: (data: AgentTokenCreatePayload): Promise => + request.post('/system/agent-token', data), + setStatus: (id: string, status: boolean): Promise => + request.put('/system/agent-token/status', { id, status }), + remove: (id: string) => request.delete(`/system/agent-token/${id}`), +} diff --git a/frontend/src/components/layout/AgentToken.vue b/frontend/src/components/layout/AgentToken.vue new file mode 100644 index 00000000..d814ee60 --- /dev/null +++ b/frontend/src/components/layout/AgentToken.vue @@ -0,0 +1,415 @@ + + + + + diff --git a/frontend/src/components/layout/Person.vue b/frontend/src/components/layout/Person.vue index ac689b12..4c368bfa 100644 --- a/frontend/src/components/layout/Person.vue +++ b/frontend/src/components/layout/Person.vue @@ -15,6 +15,7 @@ import icon_done_outlined from '@/assets/svg/icon_done_outlined.svg' import { useI18n } from 'vue-i18n' import PwdForm from './PwdForm.vue' import Apikey from './Apikey.vue' +import AgentToken from './AgentToken.vue' import { useRouter } from 'vue-router' import { useUserStore } from '@/stores/user' import { userApi } from '@/api/auth' @@ -48,6 +49,7 @@ const platFlag = computed(() => { }) const dialogVisible = ref(false) const apikeyDialogVisible = ref(false) +const agentTokenDialogVisible = ref(false) const aboutRef = ref() const languageList = computed(() => [ { @@ -102,6 +104,10 @@ const closePwd = () => { const openApikey = () => { apikeyDialogVisible.value = true } +const openAgentToken = () => { + popoverRef.value?.hide() + agentTokenDialogVisible.value = true +} const toAbout = () => { aboutRef.value?.open() } @@ -158,6 +164,12 @@ const logout = async () => {
API Key
+
+ + + +
{{ $t('agent_token.title') }}
+
diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index bb7208c4..6802c18b 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -1057,5 +1057,27 @@ "create": "Create", "to_doc": "View API", "trigger_limit": "Supports up to {0} API Keys" + }, + "agent_token": { + "title": "Agent tokens", + "info_tips": "Agent tokens let external agents such as Claude or Codex query data through sqlbot-mcp. They act with your identity and permissions and can only call sqlbot-mcp query tools. A token is shown only once when created; keep it safe and disable or delete it immediately if it leaks.", + "create": "Create token", + "name": "Name", + "name_placeholder": "e.g. My Claude", + "name_required": "Enter a token name", + "token": "Token", + "scope": "Datasources", + "scope_all": "All datasources I can access", + "scope_selected": "Selected datasources only", + "datasource_placeholder": "Select datasources", + "datasource_required": "Select at least one datasource", + "last_used": "Last used", + "never_used": "Never", + "trigger_limit": "You can create at most {0} agent tokens", + "delete_confirm": "Agents using this token will lose access immediately. Delete token {msg}?", + "created_title": "Token created", + "created_tip": "Copy and store the token now. It will not be shown again:", + "claude_config": "Add the MCP server in Claude Code:", + "done": "I have saved it" } } diff --git a/frontend/src/i18n/ko-KR.json b/frontend/src/i18n/ko-KR.json index e6d863ba..6ae17f25 100644 --- a/frontend/src/i18n/ko-KR.json +++ b/frontend/src/i18n/ko-KR.json @@ -1057,5 +1057,27 @@ "create": "생성", "to_doc": "API 보기", "trigger_limit": "최대 {0}개의 API 키 생성 지원" + }, + "agent_token": { + "title": "에이전트 토큰", + "info_tips": "에이전트 토큰을 사용하면 Claude, Codex 같은 외부 에이전트가 sqlbot-mcp로 데이터를 조회할 수 있습니다. 토큰은 사용자의 신원과 권한으로 동작하며 sqlbot-mcp 조회 도구만 호출할 수 있습니다. 토큰은 생성할 때 한 번만 표시되므로 안전하게 보관하고, 유출되면 즉시 비활성화하거나 삭제하세요.", + "create": "토큰 생성", + "name": "이름", + "name_placeholder": "예: 내 Claude", + "name_required": "토큰 이름을 입력하세요", + "token": "토큰", + "scope": "데이터 소스 범위", + "scope_all": "내가 접근할 수 있는 모든 데이터 소스", + "scope_selected": "선택한 데이터 소스만", + "datasource_placeholder": "데이터 소스 선택", + "datasource_required": "데이터 소스를 하나 이상 선택하세요", + "last_used": "마지막 사용", + "never_used": "사용 기록 없음", + "trigger_limit": "에이전트 토큰은 최대 {0}개까지 만들 수 있습니다", + "delete_confirm": "이 토큰을 사용하는 에이전트는 즉시 접근할 수 없게 됩니다. 토큰 {msg}을(를) 삭제할까요?", + "created_title": "토큰이 생성되었습니다", + "created_tip": "지금 토큰을 복사해 보관하세요. 다시 표시되지 않습니다:", + "claude_config": "Claude Code에 MCP 서버 추가:", + "done": "저장했습니다" } } diff --git a/frontend/src/i18n/th-TH.json b/frontend/src/i18n/th-TH.json index a1fa5623..de7de1d7 100644 --- a/frontend/src/i18n/th-TH.json +++ b/frontend/src/i18n/th-TH.json @@ -1057,5 +1057,27 @@ "create": "สร้าง", "to_doc": "ดู API", "trigger_limit": "รองรับการสร้าง API Key ได้สูงสุด {0} คีย์" + }, + "agent_token": { + "title": "โทเค็นเอเจนต์", + "info_tips": "โทเค็นเอเจนต์ช่วยให้เอเจนต์ภายนอก เช่น Claude หรือ Codex สืบค้นข้อมูลผ่าน sqlbot-mcp โดยใช้ตัวตนและสิทธิ์ของคุณ และเรียกได้เฉพาะเครื่องมือสืบค้นของ sqlbot-mcp โทเค็นจะแสดงเพียงครั้งเดียวตอนสร้าง โปรดเก็บรักษาให้ปลอดภัย และปิดใช้งานหรือลบทันทีหากรั่วไหล", + "create": "สร้างโทเค็น", + "name": "ชื่อ", + "name_placeholder": "เช่น Claude ของฉัน", + "name_required": "กรุณาระบุชื่อโทเค็น", + "token": "โทเค็น", + "scope": "ขอบเขตแหล่งข้อมูล", + "scope_all": "แหล่งข้อมูลทั้งหมดที่ฉันเข้าถึงได้", + "scope_selected": "เฉพาะแหล่งข้อมูลที่เลือก", + "datasource_placeholder": "เลือกแหล่งข้อมูล", + "datasource_required": "เลือกแหล่งข้อมูลอย่างน้อยหนึ่งรายการ", + "last_used": "ใช้ล่าสุด", + "never_used": "ยังไม่เคยใช้", + "trigger_limit": "สร้างโทเค็นเอเจนต์ได้สูงสุด {0} รายการ", + "delete_confirm": "เอเจนต์ที่ใช้โทเค็นนี้จะเข้าถึงไม่ได้ทันที ต้องการลบโทเค็น {msg} หรือไม่", + "created_title": "สร้างโทเค็นแล้ว", + "created_tip": "คัดลอกและเก็บโทเค็นไว้ตอนนี้ ระบบจะไม่แสดงอีก:", + "claude_config": "เพิ่มเซิร์ฟเวอร์ MCP ใน Claude Code:", + "done": "บันทึกแล้ว" } } diff --git a/frontend/src/i18n/zh-CN.json b/frontend/src/i18n/zh-CN.json index 65398fb3..c5048cce 100644 --- a/frontend/src/i18n/zh-CN.json +++ b/frontend/src/i18n/zh-CN.json @@ -1057,5 +1057,27 @@ "create": "创建", "to_doc": "查看 API", "trigger_limit": "最多支持创建 {0} 个 API Key" + }, + "agent_token": { + "title": "Agent 令牌", + "info_tips": "Agent 令牌供 Claude、Codex 等外部 Agent 通过 sqlbot-mcp 查询数据,以你的身份和权限访问,只能调用 sqlbot-mcp 的查询工具。令牌只在创建时显示一次,请妥善保管;泄露后请立即停用或删除。", + "create": "创建令牌", + "name": "名称", + "name_placeholder": "例如:我的 Claude", + "name_required": "请输入令牌名称", + "token": "令牌", + "scope": "数据源范围", + "scope_all": "跟随我的全部数据源权限", + "scope_selected": "仅限选定的数据源", + "datasource_placeholder": "选择数据源", + "datasource_required": "请至少选择一个数据源", + "last_used": "最近使用", + "never_used": "从未使用", + "trigger_limit": "最多支持创建 {0} 个 Agent 令牌", + "delete_confirm": "删除后使用该令牌的 Agent 将立即无法访问,是否删除令牌:{msg}?", + "created_title": "令牌已创建", + "created_tip": "请立即复制并保存令牌,关闭后将无法再次查看:", + "claude_config": "在 Claude Code 中添加 MCP:", + "done": "我已保存" } } diff --git a/frontend/src/i18n/zh-TW.json b/frontend/src/i18n/zh-TW.json index b2a47218..ee0aa1b1 100644 --- a/frontend/src/i18n/zh-TW.json +++ b/frontend/src/i18n/zh-TW.json @@ -1057,5 +1057,27 @@ "create": "建立", "to_doc": "檢視 API", "trigger_limit": "最多支援建立 {0} 個 API Key" + }, + "agent_token": { + "title": "Agent 權杖", + "info_tips": "Agent 權杖供 Claude、Codex 等外部 Agent 透過 sqlbot-mcp 查詢資料,以你的身分和權限存取,只能呼叫 sqlbot-mcp 的查詢工具。權杖只在建立時顯示一次,請妥善保管;外洩後請立即停用或刪除。", + "create": "建立權杖", + "name": "名稱", + "name_placeholder": "例如:我的 Claude", + "name_required": "請輸入權杖名稱", + "token": "權杖", + "scope": "資料來源範圍", + "scope_all": "跟隨我的全部資料來源權限", + "scope_selected": "僅限選定的資料來源", + "datasource_placeholder": "選擇資料來源", + "datasource_required": "請至少選擇一個資料來源", + "last_used": "最近使用", + "never_used": "從未使用", + "trigger_limit": "最多支援建立 {0} 個 Agent 權杖", + "delete_confirm": "刪除後使用該權杖的 Agent 將立即無法存取,是否刪除權杖:{msg}?", + "created_title": "權杖已建立", + "created_tip": "請立即複製並保存權杖,關閉後將無法再次查看:", + "claude_config": "在 Claude Code 中新增 MCP:", + "done": "我已保存" } } From eab46698426588d853eb69fc33c50e7b183c3a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=A2=81=E6=B7=8B=E6=B6=9B?= Date: Mon, 16 Nov 2026 11:17:20 +0800 Subject: [PATCH 3/3] =?UTF-8?q?test(mcp):=20sqlbot-mcp=20=E9=9B=86?= =?UTF-8?q?=E6=88=90=E6=B5=8B=E8=AF=95=E4=BD=BF=E7=94=A8=E7=8B=AC=E7=AB=8B?= =?UTF-8?q?=E7=9A=84=E6=95=B0=E6=8D=AE=E6=BA=90=E5=AD=98=E5=82=A8=E5=AF=86?= =?UTF-8?q?=E9=92=A5=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 数据源配置改为部署专用密钥加密后,测试进程与子服务进程需要共享同一个临时密钥文件。 AI-Coding: true AI-Tool: TophantAI --- backend/tests/test_sqlbot_mcp_integration.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_sqlbot_mcp_integration.py b/backend/tests/test_sqlbot_mcp_integration.py index c848c5c8..cf9e59b6 100644 --- a/backend/tests/test_sqlbot_mcp_integration.py +++ b/backend/tests/test_sqlbot_mcp_integration.py @@ -46,10 +46,20 @@ def serve_controlled_process(app_name, port): @pytest.fixture -def environment(tmp_path): +def environment(tmp_path, monkeypatch): + import base64 + import secrets + url = make_url(os.environ["SQLBOT_TEST_DATABASE_URL"]) + # Datasource configurations are encrypted with a per-deployment key; this + # process (seeding) and the child server must share one throwaway key file. + keys_file = tmp_path / "datasource-keys.json" + keys_file.write_text(json.dumps({"active": "it", "keys": {"it": base64.b64encode( + secrets.token_bytes(32)).decode()}})) + monkeypatch.setenv("SQLBOT_DATASOURCE_KEYS_FILE", str(keys_file)) env = dict(os.environ) env.update({ + "SQLBOT_DATASOURCE_KEYS_FILE": str(keys_file), "POSTGRES_SERVER": url.host or "127.0.0.1", "POSTGRES_PORT": str(url.port or 5432), "POSTGRES_USER": url.username or "", @@ -115,7 +125,7 @@ def _seed(env_info): conf = {"host": url.host, "port": url.port or 5432, "username": url.username, "password": url.password, "database": url.database, "dbSchema": "hr", "timeout": 10} DatasourceConf(**conf) - # aes_encrypt returns bytes; datasources store the base64 text. + # aes_encrypt returns bytes; datasources store the text form. configuration = aes_encrypt(json.dumps(conf)).decode() hr = CoreDatasource(name="hr", description="hr data", type="pg", type_name="PostgreSQL", configuration=configuration, oid=10, create_by=1, status="Success",