From 9f2a6fdefe43205cb6ad10c099b984e583e8cc93 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Mon, 16 Nov 2026 12:09:11 -0400 Subject: [PATCH] fix(datasource): convert CSV datetime columns after parsing --- backend/apps/datasource/utils/excel.py | 7 +++ backend/tests/test_csv_datetime_import.py | 53 +++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 backend/tests/test_csv_datetime_import.py diff --git a/backend/apps/datasource/utils/excel.py b/backend/apps/datasource/utils/excel.py index 2d982097c..b325a205f 100644 --- a/backend/apps/datasource/utils/excel.py +++ b/backend/apps/datasource/utils/excel.py @@ -52,7 +52,14 @@ def read_import_dataframe(save_path: str, sheet_name: str, field_mapping: dict): for column, field_type in field_mapping.items() } if save_path.endswith('.csv'): + # The CSV parser cannot read datetime64 dtypes directly. + datetime_columns = [column for column, field_type in field_mapping.items() if field_type == 'datetime'] + for column in datetime_columns: + dtype_dict[column] = 'string' df = pd.read_csv(save_path, engine='c', dtype=dtype_dict) + for column in datetime_columns: + if column in df.columns: + df[column] = df[column].astype(USER_TYPE_TO_PANDAS['datetime']) else: df = pd.read_excel(save_path, sheet_name=sheet_name, engine='calamine', dtype=dtype_dict) for column, field_type in field_mapping.items(): diff --git a/backend/tests/test_csv_datetime_import.py b/backend/tests/test_csv_datetime_import.py new file mode 100644 index 000000000..272be335f --- /dev/null +++ b/backend/tests/test_csv_datetime_import.py @@ -0,0 +1,53 @@ +"""Regression tests for explicitly selected CSV datetime columns.""" + +import sys +import tempfile +import unittest +from pathlib import Path + +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from apps.datasource.utils.excel import read_import_dataframe + + +class CsvDatetimeImportTests(unittest.TestCase): + def read_csv(self, contents, mapping): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "import.csv" + path.write_text(contents, encoding="utf-8") + return read_import_dataframe(str(path), "Sheet1", mapping) + + def test_dates_and_empty_cells(self): + result = self.read_csv( + "created,amount,label\n2026-11-16,1.5,first\n,2.5,second\n", + {"created": "datetime", "amount": "int", "label": "string"}, + ) + self.assertEqual(str(result["created"].dtype), "datetime64[ns]") + self.assertEqual(result["created"][0], pd.Timestamp("2026-11-16")) + self.assertTrue(pd.isna(result["created"][1])) + self.assertEqual(result["amount"].tolist(), [2, 3]) + self.assertEqual(result["label"].tolist(), ["first", "second"]) + + def test_time_components_are_preserved(self): + result = self.read_csv( + "created\n2026-11-16 12:34:56\n", {"created": "datetime"} + ) + self.assertEqual(result["created"][0], pd.Timestamp("2026-11-16 12:34:56")) + + def test_invalid_dates_are_rejected(self): + with self.assertRaises(ValueError): + self.read_csv("created\nnot-a-date\n", {"created": "datetime"}) + + def test_date_strings_remain_strings_when_selected(self): + result = self.read_csv("created\n2026-11-16\n", {"created": "string"}) + self.assertEqual(result["created"][0], "2026-11-16") + + def test_empty_csv_with_headers(self): + result = self.read_csv("created\n", {"created": "datetime"}) + self.assertTrue(result.empty) + self.assertEqual(str(result["created"].dtype), "datetime64[ns]") + + +if __name__ == "__main__": + unittest.main()