Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions backend/apps/datasource/crud/datasource.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import datetime
import json
import re
from typing import List, Optional

from fastapi import HTTPException
from sqlalchemy import and_, text
from sqlalchemy import MetaData, Table, and_, text
from sqlalchemy.schema import DropTable
from sqlbot_xpack.permissions.models.ds_rules import DsRules
from sqlmodel import select

Expand Down Expand Up @@ -44,6 +46,22 @@ def get_ds(session: SessionDep, id: int):
return datasource


def validate_excel_sheet_names(configuration: dict) -> list[str]:
"""Reject SQL syntax and non-import table names, including stored legacy input."""
sheets = configuration.get("sheets", [])
if not isinstance(sheets, list):
raise DatasourceConfigurationError("Invalid Excel sheets")
names = []
for sheet in sheets:
name = sheet.get("tableName") if isinstance(sheet, dict) else None
if not isinstance(name, str) or re.fullmatch(
r"excel_[\u4e00-\u9fa5a-zA-Z0-9]*_[0-9a-f]{10}", name
) is None:
raise DatasourceConfigurationError("Invalid Excel table name")
names.append(name)
return names


def seal_browser_configuration(configuration: str, previous: str | None = None) -> str:
incoming = parse_configuration(decode_legacy_configuration(configuration))
if previous:
Expand All @@ -58,6 +76,8 @@ def seal_browser_configuration(configuration: str, previous: str | None = None)
if retained and any(candidate[field] != existing[field] for field in destination):
raise DatasourceConfigurationError("Connection settings changed while retaining credentials")
incoming = {**saved, **incoming}
if "sheets" in incoming:
validate_excel_sheet_names(incoming)
return aes_encrypt(json.dumps(incoming)).decode("ascii")


Expand Down Expand Up @@ -165,11 +185,12 @@ async def delete_ds(session: SessionDep, id: int):
term = session.exec(select(CoreDatasource).where(CoreDatasource.id == id)).first()
if term.type == "excel":
# drop all tables for current datasource
names = validate_excel_sheet_names(json.loads(aes_decrypt(term.configuration)))
engine = get_engine_conn()
conf = DatasourceConf(**json.loads(aes_decrypt(term.configuration)))
with engine.connect() as conn:
for sheet in conf.sheets:
conn.execute(text(f'DROP TABLE IF EXISTS "{sheet["tableName"]}"'))
for name in names:
table = Table(name, MetaData(), quote=True)
conn.execute(DropTable(table, if_exists=True))
conn.commit()

session.delete(term)
Expand Down
16 changes: 8 additions & 8 deletions backend/tests/test_datasource_configuration_crypto.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,8 +224,8 @@ def edit_before_write(statement, *args, **kwargs):
def test_datasource_http_postgres(pg, monkeypatch, caplog):
database, schema = pg
with database.begin() as connection:
connection.execute(text(f"CREATE TABLE {schema}.synthetic_sample (id integer)"))
connection.execute(text(f"INSERT INTO {schema}.synthetic_sample VALUES (42)"))
connection.execute(text(f"CREATE TABLE {schema}.excel_sample_0123456789 (id integer)"))
connection.execute(text(f"INSERT INTO {schema}.excel_sample_0123456789 VALUES (42)"))
monkeypatch.setattr(permission, "engine", database)
monkeypatch.setattr(datasource, "run_save_ds_embeddings", lambda *_: None)
monkeypatch.setattr(datasource, "run_save_table_embeddings", lambda *_: None)
Expand Down Expand Up @@ -281,9 +281,9 @@ def post(path, **kwargs):
assert post("/check", json=request).json() is True
tables = post("/getTablesByConf", json=request)
assert tables.status_code == 200, tables.text
assert "synthetic_sample" in tables.text
assert "excel_sample_0123456789" in tables.text
added = post(
"/add", json={**request, "tables": [{"table_name": "synthetic_sample"}]}
"/add", json={**request, "tables": [{"table_name": "excel_sample_0123456789"}]}
)
assert added.status_code == 200, added.text
ds_id = added.json()["id"]
Expand Down Expand Up @@ -321,14 +321,14 @@ def post(path, **kwargs):
assert saved != stored
assert json.loads(aes_decrypt(saved))["password"] == "synthetic-only"
assert client.get(f"/datasource/check/{ds_id}").json() is True
assert post(f"/getFields/{ds_id}/synthetic_sample").status_code == 200
assert post(f"/getFields/{ds_id}/excel_sample_0123456789").status_code == 200
table = post(f"/tableList/{ds_id}").json()[0]
preview = post(f"/previewData/{ds_id}", json={"table": table})
assert preview.status_code == 200 and "42" in preview.text, preview.text
with Session(database) as session:
ds = session.get(CoreDatasource, ds_id)
try:
assert exec_sql(ds, "select id from synthetic_sample")["data"] == [
assert exec_sql(ds, "select id from excel_sample_0123456789")["data"] == [
{"id": 42}
]
finally:
Expand All @@ -343,7 +343,7 @@ def post(path, **kwargs):

excel_config = {
"filename": "synthetic.xlsx",
"sheets": [{"tableName": "synthetic_sample"}],
"sheets": [{"tableName": "excel_sample_0123456789"}],
}
excel = post(
"/add",
Expand All @@ -358,7 +358,7 @@ def post(path, **kwargs):
with Session(database) as session:
excel_ds = session.get(CoreDatasource, excel.json()["id"])
assert json.loads(aes_decrypt(excel_ds.configuration)) == excel_config
assert "synthetic_sample" in post(f"/getTables/{excel_ds.id}").text
assert "excel_sample_0123456789" in post(f"/getTables/{excel_ds.id}").text
assistant = AssistantOutDsSchema(
name="synthetic", type="pg", password="assistant-only"
)
Expand Down
55 changes: 55 additions & 0 deletions backend/tests/test_excel_drop_injection.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
"""Offline regression for untrusted persisted Excel identifiers."""
import ast
import re
from pathlib import Path

import pytest

SOURCE = Path(__file__).resolve().parents[1] / 'apps/datasource/crud/datasource.py'


def validator():
tree = ast.parse(SOURCE.read_text())
function = next(n for n in tree.body if isinstance(n, ast.FunctionDef)
and n.name == 'validate_excel_sheet_names')
scope = {'re': re, 'DatasourceConfigurationError': ValueError}
exec(compile(ast.Module(body=[function], type_ignores=[]), str(SOURCE), 'exec'), scope)
return scope['validate_excel_sheet_names']


@pytest.mark.parametrize('name', ['x"; SELECT 1; --', 'users', '', None,
'excel_x_0123456789"', 'public.excel_x_0123456789'])
def test_reject_untrusted_table_names(name):
with pytest.raises(ValueError):
validator()({'sheets': [{'tableName': name}]})


def test_accept_generated_names():
names = ['excel_销售_0123456789', 'excel__abcdef0123']
assert validator()({'sheets': [{'tableName': name} for name in names]}) == names


@pytest.mark.parametrize('sheets', [None, {}, ['bad'], [{}]])
def test_reject_malformed_sheets(sheets):
with pytest.raises(ValueError):
validator()({'sheets': sheets})


def test_write_and_delete_validate_and_quote():
source = SOURCE.read_text()
assert 'validate_excel_sheet_names(incoming)' in source
assert 'validate_excel_sheet_names(json.loads(aes_decrypt(term.configuration)))' in source
assert 'Table(name, MetaData(), quote=True)' in source
assert 'conn.execute(DropTable(table, if_exists=True))' in source
assert 'sheet["tableName"]' not in source


@pytest.mark.parametrize('name', ['excel_销售_0123456789', 'x"; SELECT 1; --', 'public.users'])
def test_drop_table_compiles_name_as_single_identifier(name):
from sqlalchemy import MetaData, Table
from sqlalchemy.dialects import postgresql
from sqlalchemy.schema import DropTable

ddl = DropTable(Table(name, MetaData(), quote=True), if_exists=True)
compiled = str(ddl.compile(dialect=postgresql.dialect())).strip()
assert compiled == 'DROP TABLE IF EXISTS "' + name.replace('"', '""') + '"'
Loading