diff --git a/CHANGELOG.md b/CHANGELOG.md index b69f1d504..234877da8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Release History # Unreleased +- Kernel HTTPS traffic negotiates HTTP/2 with HTTP/1.1 fallback. Set `enable_http2=False` to force HTTP/1.1. Configured proxies also force HTTP/1.1. - Honor `max_download_threads` on kernel connections, with a default download window of 16 and explicit chunk-setting overrides. Invalid values raise `ValueError` when connecting with `use_kernel=True`. - Fix the kernel backend ignoring `use_cloud_fetch=False`, so complete results can be fetched without cloud-storage downloads. - Fix inline parameters (`use_inline_params=True`) so a sequence directly after `IN` renders as a list again: `x IN %(ids)s` renders as `x IN (1,2,3)` instead of `x IN ARRAY(1,2,3)`, which Spark rejects. Inside the list, tuples render as rows for a multi-column `IN`, and a `set` or `frozenset` is accepted again. Sequences anywhere else still render as `ARRAY(...)` ([#629](https://github.com/databricks/databricks-sql-python/issues/629)) diff --git a/README.md b/README.md index 36162e69b..595505c59 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Install using `pip install databricks-sql-connector` Install using `pip install databricks-sql-connector[pyarrow]` ### Installing with the Rust kernel backend (`use_kernel=True`) + Install using `pip install databricks-sql-connector[kernel]` This adds the optional [`databricks-sql-kernel`](https://pypi.org/project/databricks-sql-kernel/) @@ -52,6 +53,9 @@ connection = sql.connect( ``` Notes: +- HTTPS uses HTTP/2 with HTTP/1.1 fallback. `enable_http2=False` or configured + proxy environment variables force HTTP/1.1 for all kernel traffic, including + bypassed destinations. - Requires **Python >= 3.10** (the kernel wheel is published as `cp310-abi3`). On older interpreters the `[kernel]` extra installs nothing and `use_kernel=True` raises an `ImportError`. diff --git a/src/databricks/sql/backend/kernel/client.py b/src/databricks/sql/backend/kernel/client.py index eab9847ca..58821cc98 100644 --- a/src/databricks/sql/backend/kernel/client.py +++ b/src/databricks/sql/backend/kernel/client.py @@ -218,6 +218,14 @@ def _kernel_session_accepts_kwarg(name: str) -> bool: return support.accepts_kwargs or name in support.parameters +def _kernel_http2_kwargs(value: Optional[bool]) -> Dict[str, bool]: + if value is None: + return {} + if not isinstance(value, bool): + raise ValueError("enable_http2 must be a bool") + return {"enable_http2": value} + + def _kernel_geospatial_kwargs(value: bool) -> Dict[str, bool]: """Build the explicit native geospatial-support kwarg. @@ -346,6 +354,7 @@ def __init__( # The kernel binding owns type and range validation. self._request_timeout_secs = kwargs.get("request_timeout_secs") self._max_connections = kwargs.get("max_connections") + self._http2_kwargs = _kernel_http2_kwargs(kwargs.get("enable_http2")) self._use_cloud_fetch = kwargs.get("use_cloud_fetch", True) self._max_download_threads = kwargs.get("max_download_threads", 16) if ( @@ -538,6 +547,7 @@ def open_session( **geospatial_kwargs, **max_connections_kwargs, **http_headers_kwargs, + **self._http2_kwargs, ) except Exception as exc: raise _wrap_kernel_exception("open_session", exc) from exc diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py index d8240efaf..6d644324a 100755 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -193,6 +193,10 @@ def __init__( decision. This is an intentional divergence from the Thrift/SEA paths, where an explicit ``True`` can still be suppressed by the feature flag. + :param enable_http2: `bool`, optional (kernel default is True) + Kernel backend only. Negotiate HTTP/2 for all kernel HTTPS + traffic, with HTTP/1.1 fallback. ``False`` or configured proxy + environment variables force HTTP/1.1, including bypassed hosts. :param enable_geospatial_support: `bool`, optional (default is False) Kernel backend only. Controls the public representation of ``GEOMETRY`` and ``GEOGRAPHY`` result values. ``True`` returns diff --git a/src/databricks/sql/session.py b/src/databricks/sql/session.py index cd91693d3..23693141c 100644 --- a/src/databricks/sql/session.py +++ b/src/databricks/sql/session.py @@ -337,6 +337,7 @@ def _create_backend( retry_options=kernel_retry_options, request_timeout_secs=kwargs.get("_socket_timeout"), max_connections=kwargs.get("_pool_maxsize") or None, + enable_http2=kwargs.get("enable_http2"), use_cloud_fetch=kwargs.get("use_cloud_fetch", True), max_download_threads=kwargs.get("max_download_threads", 16), enable_geospatial_support=kwargs.get( diff --git a/tests/unit/test_kernel_client.py b/tests/unit/test_kernel_client.py index 5bad2040b..819cdcbb6 100644 --- a/tests/unit/test_kernel_client.py +++ b/tests/unit/test_kernel_client.py @@ -407,6 +407,52 @@ def fake_session(**options): assert configuration == original +@pytest.mark.parametrize("enabled", [None, True, False]) +def test_open_session_preserves_http2_option(monkeypatch, enabled): + captured = {} + + def fake_session(**kwargs): + captured.update(kwargs) + session = MagicMock() + session.session_id = "sess-id" + return session + + monkeypatch.setattr(kernel_client._kernel, "Session", fake_session) + client = kernel_client.KernelDatabricksClient( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + auth_provider=AccessTokenAuthProvider("dapi-test"), + ssl_options=None, + **({"enable_http2": enabled} if enabled is not None else {}), + ) + client.open_session(session_configuration=None, catalog=None, schema=None) + if enabled is None: + assert "enable_http2" not in captured + else: + assert captured["enable_http2"] is enabled + + +@pytest.mark.parametrize("value", ["false", 0, 1]) +def test_http2_option_rejects_non_boolean_through_connect(monkeypatch, value): + from databricks import sql + + native_session = MagicMock() + auth_kwargs = MagicMock() + monkeypatch.setattr(kernel_client._kernel, "Session", native_session) + monkeypatch.setattr(kernel_client, "kernel_auth_kwargs", auth_kwargs) + with pytest.raises(ValueError, match="enable_http2 must be a bool"): + sql.connect( + server_hostname="example.cloud.databricks.com", + http_path="/sql/1.0/warehouses/abc", + access_token="dapi-test", + use_kernel=True, + enable_telemetry=False, + enable_http2=value, + ) + native_session.assert_not_called() + auth_kwargs.assert_not_called() + + @pytest.mark.parametrize("timeout", [None, 0, 12.5]) def test_open_session_passes_request_timeout_to_kernel(monkeypatch, timeout): captured = {} diff --git a/tests/unit/test_session.py b/tests/unit/test_session.py index fb889d856..08b6668a7 100644 --- a/tests/unit/test_session.py +++ b/tests/unit/test_session.py @@ -540,6 +540,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _pool_maxsize=41, use_cloud_fetch=False, max_download_threads=4, + enable_http2=False, enable_geospatial_support=False, ) try: @@ -553,6 +554,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): assert kwargs["max_connections"] == 41 assert kwargs["use_cloud_fetch"] is False assert kwargs["max_download_threads"] == 4 + assert kwargs["enable_http2"] is False assert kwargs["enable_geospatial_support"] is False finally: conn.close() @@ -569,6 +571,7 @@ def test_retry_and_socket_timeout_threaded_into_kernel_client(self): _, kwargs = mock_kernel_client.call_args assert kwargs["max_connections"] is None assert kwargs["max_download_threads"] == 16 + assert kwargs["enable_http2"] is None finally: conn.close()