diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index 2e964fe0..05cfe492 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -34,10 +34,10 @@ from ..crud.table import get_tables_by_ds_id from ..models.datasource import CoreDatasource, CreateDatasource, TableObj, CoreTable, CoreField, FieldObj, \ TableSchemaResponse, ColumnSchemaResponse, PreviewResponse, ImportRequest +from ..utils.excel import is_csv_file, parse_excel_preview, read_import_dataframe from ..utils.utils import ( DatasourceConfigurationError, DatasourceKeyError, aes_decrypt, parse_configuration, ) -from ..utils.excel import parse_excel_preview, read_import_dataframe router = APIRouter(tags=["Datasource"], prefix="/datasource") path = settings.EXCEL_PATH @@ -416,7 +416,7 @@ async def upload_excel(trans: Trans, session: SessionDep, def inner(): sheets = [] engine = get_engine_conn() - if filename.endswith(".csv"): + if is_csv_file(filename): df = pd.read_csv(save_path, engine='c') tableName = f"sheet1_{hashlib.sha256(uuid.uuid4().bytes).hexdigest()[:10]}" sheets.append({"tableName": tableName, "tableComment": ""}) @@ -686,7 +686,7 @@ def inner(): field_mapping = {f.fieldName: f.fieldType for f in fields} try: df = read_import_dataframe(save_path, sheet_name, field_mapping) - if save_path.endswith(".csv"): + if is_csv_file(save_path): sheet_name = "Sheet1" except Exception as e: raise HTTPException(500, f"{trans('i18n_ds_upload_error')}: {str(e)}") diff --git a/backend/apps/datasource/utils/excel.py b/backend/apps/datasource/utils/excel.py index b325a205..114751cd 100644 --- a/backend/apps/datasource/utils/excel.py +++ b/backend/apps/datasource/utils/excel.py @@ -22,6 +22,10 @@ } +def is_csv_file(file_path: str) -> bool: + return file_path.lower().endswith(".csv") + + def infer_field_type(dtype) -> str: dtype_str = str(dtype) return FIELD_TYPE_MAP.get(dtype_str, 'string') @@ -51,6 +55,7 @@ def read_import_dataframe(save_path: str, sheet_name: str, field_mapping: dict): column: 'object' if field_type == 'int' else USER_TYPE_TO_PANDAS.get(field_type, 'string') for column, field_type in field_mapping.items() } + if is_csv_file(save_path): if save_path.endswith('.csv'): # The CSV parser cannot read datetime64 dtypes directly. datetime_columns = [column for column, field_type in field_mapping.items() if field_type == 'datetime'] @@ -75,7 +80,7 @@ def read_import_dataframe(save_path: str, sheet_name: str, field_mapping: dict): def parse_excel_preview(save_path: str, max_rows: int = 10): sheets_data = [] - if save_path.endswith(".csv"): + if is_csv_file(save_path): df = pd.read_csv(save_path, engine='c') fields = [] for col in df.columns: diff --git a/backend/tests/test_datasource_csv_extension.py b/backend/tests/test_datasource_csv_extension.py new file mode 100644 index 00000000..d2714ece --- /dev/null +++ b/backend/tests/test_datasource_csv_extension.py @@ -0,0 +1,39 @@ +import pandas as pd +import pytest + +from apps.datasource.utils.excel import parse_excel_preview, read_import_dataframe + + +@pytest.fixture(params=["csv", "CSV", "CsV"]) +def stored_csv(tmp_path, request): + path = tmp_path / f"sales_0123456789.{request.param}" + path.write_text("region,units\nEast,3\nWest,7\n", encoding="utf-8") + return str(path) + + +def test_preview_csv_with_any_extension_case(stored_csv): + [sheet] = parse_excel_preview(stored_csv) + assert sheet["sheetName"] == "Sheet1" + assert sheet["rows"] == 2 + assert sheet["data"] == [ + {"region": "East", "units": 3}, + {"region": "West", "units": 7}, + ] + + +def test_import_stored_csv_with_any_extension_case(stored_csv): + frame = read_import_dataframe( + stored_csv, "Sheet1", {"region": "string", "units": "int"} + ) + assert frame["region"].tolist() == ["East", "West"] + assert frame["units"].tolist() == [3, 7] + + +def test_excel_reader_preserves_named_sheet(tmp_path): + path = tmp_path / "sales.CSV.xlsx" + pd.DataFrame({"units": [3]}).to_excel(path, sheet_name="Sales", index=False) + + [sheet] = parse_excel_preview(str(path)) + assert sheet["sheetName"] == "Sales" + frame = read_import_dataframe(str(path), "Sales", {"units": "int"}) + assert frame["units"].tolist() == [3] diff --git a/backend/tests/test_datasource_upload_path_traversal.py b/backend/tests/test_datasource_upload_path_traversal.py index 3505bc78..3ad2a6dd 100644 --- a/backend/tests/test_datasource_upload_path_traversal.py +++ b/backend/tests/test_datasource_upload_path_traversal.py @@ -122,3 +122,15 @@ def test_normal_names_keep_their_own_stem_and_extension(parse_harness): stored = response.json()["filePath"] assert stored.startswith("a.b_") and stored.endswith(".xlsx"), stored assert (parse_harness.upload_dir / stored).is_file() + + +def test_legacy_upload_imports_mixed_case_csv(harness, monkeypatch): + insert = Mock() + monkeypatch.setattr(datasource, "insert_pg", insert) + response = harness.client.post( + UPLOAD_URL, files={"file": ("orders.CsV", b"units\n3\n7\n", "text/csv")} + ) + assert response.status_code == 200, response.text + insert.assert_called_once() + frame = insert.call_args.args[0] + assert frame.to_dict("records") == [{"units": 3}, {"units": 7}]