From ff29d66297694e4d693b29530999ddfb756a2aa9 Mon Sep 17 00:00:00 2001 From: Shahzeb Ali <126076773+Shahzeb-DataEngineer@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:25:08 +0500 Subject: [PATCH] fix: accept up to 9 fractional digits for micros timestamps Widen ISO_TIMESTAMP and ISO_TIMESTAMPTZ to accept 1-9 fractional digits and zone offsets with optional seconds, and parse micros timestamps through the nanosecond path, truncating to microseconds, mirroring the Java implementation. Previously, valid ISO-8601 strings with nanosecond precision were rejected with a ValueError when binding string literals to timestamp/timestamptz columns or parsing JSON single values. Signed-off-by: Shahzeb Ali <126076773+Shahzeb-DataEngineer@users.noreply.github.com> --- pyiceberg/utils/datetime.py | 12 +++++---- tests/test_conversions.py | 17 ++++++++++++ tests/utils/test_datetime.py | 51 ++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index ea7329ea20..70ffb1f303 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -28,11 +28,11 @@ EPOCH_DATE = date.fromisoformat("1970-01-01") EPOCH_TIMESTAMP = datetime.fromisoformat("1970-01-01T00:00:00.000000") -ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?") +ISO_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,9})?") ISO_TIMESTAMP_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?") EPOCH_TIMESTAMPTZ = datetime.fromisoformat("1970-01-01T00:00:00.000000+00:00") -ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,6})?[-+]\d{2}:\d{2}") -ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2})") +ISO_TIMESTAMPTZ = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d{1,9})?[-+]\d{2}:\d{2}(:\d{2})?") +ISO_TIMESTAMPTZ_NANO = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(.\d{1,6})?(\d{1,3})?([-+]\d{2}:\d{2}(:\d{2})?)") def micros_to_days(timestamp: int) -> int: @@ -86,7 +86,8 @@ def datetime_to_micros(dt: datetime) -> int: def timestamp_to_micros(timestamp_str: str) -> int: """Convert an ISO-9601 formatted timestamp without zone to microseconds from 1970-01-01T00:00:00.000000.""" if ISO_TIMESTAMP.fullmatch(timestamp_str): - return datetime_to_micros(datetime.fromisoformat(timestamp_str)) + # Parse through the nanosecond path and truncate to microseconds, mirroring Java + return nanos_to_micros(timestamp_to_nanos(timestamp_str)) if ISO_TIMESTAMPTZ.fullmatch(timestamp_str): # When we can match a timestamp without a zone, we can give a more specific error raise ValueError(f"Zone offset provided, but not expected: {timestamp_str}") @@ -168,7 +169,8 @@ def millis_to_datetime(millis: int) -> datetime: def timestamptz_to_micros(timestamptz_str: str) -> int: """Convert an ISO-8601 formatted timestamp with zone to microseconds from 1970-01-01T00:00:00.000000+00:00.""" if ISO_TIMESTAMPTZ.fullmatch(timestamptz_str): - return datetime_to_micros(datetime.fromisoformat(timestamptz_str)) + # Parse through the nanosecond path and truncate to microseconds, mirroring Java + return nanos_to_micros(timestamptz_to_nanos(timestamptz_str)) if ISO_TIMESTAMP.fullmatch(timestamptz_str): # When we can match a timestamp without a zone, we can give a more specific error raise ValueError(f"Missing zone offset: {timestamptz_str} (must be ISO-8601)") diff --git a/tests/test_conversions.py b/tests/test_conversions.py index e786ae0683..3e44c38eb3 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -609,6 +609,23 @@ def test_json_serialize_roundtrip(primitive_type: PrimitiveType, value: Any) -> assert value == conversions.from_json(primitive_type, conversions.to_json(primitive_type, value)) +@pytest.mark.parametrize( + "primitive_type, json_val, expected", + [ + (TimestampType(), "2017-11-16T22:31:08.123456789", datetime(2017, 11, 16, 22, 31, 8, 123456)), + ( + TimestamptzType(), + "2017-11-16T22:31:08.123456789+00:00", + datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=timezone.utc), + ), + ], +) +def test_json_single_deserialization_with_nano_precision( + primitive_type: PrimitiveType, json_val: str, expected: datetime +) -> None: + assert conversions.from_json(primitive_type, json_val) == expected + + def test_unknown_type_conversions() -> None: """Unknown values are always null, so they have no binary or partition representation.""" unknown = UnknownType() diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index 54fd3eefcc..2b4121dc93 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -28,7 +28,9 @@ nanos_to_micros, time_str_to_nanos, time_to_nanos, + timestamp_to_micros, timestamp_to_nanos, + timestamptz_to_micros, timestamptz_to_nanos, ) @@ -155,6 +157,55 @@ def test_timestamptz_to_nanos_invalid_timestamp() -> None: timestamptz_to_nanos("invalid") +@pytest.mark.parametrize( + "timestamp, micros", + [ + ("1970-01-01T00:00:00", 0), + ("2025-02-23T20:21:44.375612", 1740342104375612), + ("2025-02-23T20:21:44.3756127", 1740342104375612), + ("2025-02-23T20:21:44.37561278", 1740342104375612), + ("2025-02-23T20:21:44.375612001", 1740342104375612), + ], +) +def test_timestamp_to_micros(timestamp: str, micros: int) -> None: + assert micros == timestamp_to_micros(timestamp) + + +def test_timestamp_to_micros_unexpected_zone_offset() -> None: + with pytest.raises(ValueError, match="Zone offset provided, but not expected: 2025-02-23T16:21:44.375612001-04:00"): + timestamp_to_micros("2025-02-23T16:21:44.375612001-04:00") + + +def test_timestamp_to_micros_invalid_timestamp() -> None: + with pytest.raises(ValueError, match=re.escape("Invalid timestamp without zone: invalid (must be ISO-8601)")): + timestamp_to_micros("invalid") + + +@pytest.mark.parametrize( + "timestamp, micros", + [ + ("1970-01-01T00:00:00+00:00", 0), + ("2025-02-23T16:21:44.375612-04:00", 1740342104375612), + ("2025-02-23T16:21:44.3756127-04:00", 1740342104375612), + ("2025-02-23T16:21:44.37561278-04:00", 1740342104375612), + ("2025-02-23T16:21:44.375612001-04:00", 1740342104375612), + ("2025-02-23T16:21:44-04:00:00", 1740342104000000), + ], +) +def test_timestamptz_to_micros(timestamp: str, micros: int) -> None: + assert micros == timestamptz_to_micros(timestamp) + + +def test_timestamptz_to_micros_missing_zone_offset() -> None: + with pytest.raises(ValueError, match="Missing zone offset: 2025-02-23T20:21:44.375612001"): + timestamptz_to_micros("2025-02-23T20:21:44.375612001") + + +def test_timestamptz_to_micros_invalid_timestamp() -> None: + with pytest.raises(ValueError, match=re.escape("Invalid timestamp with zone: invalid (must be ISO-8601)")): + timestamptz_to_micros("invalid") + + @pytest.mark.parametrize("nanos, micros", [(1510871468000001001, 1510871468000001), (-1510871468000001001, -1510871468000002)]) def test_nanos_to_micros(nanos: int, micros: int) -> None: assert micros == nanos_to_micros(nanos)