Expose audio/video time_ranges citations

.github/workflows/ci.yml

@@ -8,7 +8,8 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    # ubuntu-latest (26.04) no longer ships Python 3.9, which the package supports.
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]

docs/usage.md

@@ -316,6 +316,10 @@ for chunk in result.text_chunks:
         print(f"  Summary: {summary}")
 ```
 
+Audio and video transcripts are text chunks with speaker-labelled lines
+(`S1: ...`). Their metadata adds `time_ranges` (milliseconds), `speakers`,
+`media` (`kind`, `duration_ms`) and `source_kind` (`"asr"`).
+
 ### ImageChunk
 
 | Field | Type | Description |
@@ -548,6 +552,7 @@ print(response.failure_reason)       # no-answer reason, when returned
 for ref in response.referenced_chunks:
     print(ref.chunk_id, ref.document_id, ref.chunk_type, ref.content_source)
     print(ref.section_path, ref.file_path, ref.job_id, ref.asset_url, ref.metadata)
+    print(ref.time_ranges)           # audio/video only: [TimeRange(start_ms, end_ms)]
 
 # results are raw path chunks for debug
 for result in response.results:
@@ -584,6 +589,16 @@ result.metadata   # Optional[dict]
 result.source.document_id
 result.source.source_file_name
 result.source.section_path
+result.source.time_ranges  # Optional[list[TimeRange]], audio/video only
+```
+
+Audio and video documents are speech transcripts. Their citations carry
+`time_ranges` in milliseconds, so you can seek the recording:
+
+```python
+for ref in response.referenced_chunks:
+    for span in ref.time_ranges or []:
+        print(f"{ref.section_path} @ {span.start_ms / 1000:.0f}s-{span.end_ms / 1000:.0f}s")
 ```
 
 ### Limit or exclude documents or sections
@@ -925,10 +940,12 @@ At `DEBUG` level, the SDK logs every HTTP request and response (with API keys re
 | Category | Extensions |
 |----------|-----------|
 | Documents | `.doc`, `.docx`, `.pdf`, `.txt` |
-| Spreadsheets | `.xls`, `.xlsx`, `.csv` |
-| Images | `.jpg`, `.jpeg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.svg` |
-| Presentations | `.ppt`, `.pptx` |
-| Markdown | `.md` |
+| Spreadsheets | `.xls`, `.xlsx`, `.csv`, `.tsv` |
+| Images | `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp`, `.tif`, `.tiff`, `.heic`, `.heif` |
+| Presentations | `.pptx` |
+| Markdown and web | `.md`, `.html`, `.htm` |
+| Audio | `.mp3`, `.wav`, `.m4a`, `.aac`, `.flac`, `.ogg`, `.opus` |
+| Video (audio track only) | `.mp4`, `.mov`, `.webm`, `.mkv` |
 
 Use the `doc_type` parsing parameter to override auto-detection:
 

src/knowhere/__init__.py

@@ -94,6 +94,7 @@
     RetrievalResult,
     RetrievalSectionExclusion,
     RetrievalSource,
+    TimeRange,
 )
 
 __all__: list[str] = [
@@ -150,6 +151,7 @@
     "RetrievalReferencedChunk",
     "RetrievalSectionExclusion",
     "RetrievalSource",
+    "TimeRange",
     "RetrievalEvidencePart",
     "RetrievalQueryResponse",
     "RetrievalResult",

src/knowhere/types/__init__.py

@@ -57,6 +57,7 @@
     RetrievalResult,
     RetrievalSectionExclusion,
     RetrievalSource,
+    TimeRange,
 )
 
 __all__: list[str] = [
@@ -85,6 +86,7 @@
     "RetrievalReferencedChunk",
     "RetrievalSectionExclusion",
     "RetrievalSource",
+    "TimeRange",
     "RetrievalEvidencePart",
     "RetrievalQueryResponse",
     "RetrievalResult",

src/knowhere/types/result.py

@@ -11,6 +11,7 @@
 from pydantic import BaseModel, Field
 
 from knowhere._exceptions import ValidationError
+from knowhere.types.retrieval import TimeRange
 
 
 # ---------------------------------------------------------------------------
@@ -203,6 +204,11 @@ class ChunkMetadata(BaseModel):
     original_name: Optional[str] = None
     table_type: Optional[str] = None
     document_top_summary: Optional[str] = None
+    # Audio/video transcript chunks.
+    time_ranges: Optional[List[TimeRange]] = None
+    speakers: Optional[List[str]] = None
+    media: Optional[Dict[str, Any]] = None
+    source_kind: Optional[str] = None
 
 
 class BaseChunk(BaseModel):

src/knowhere/types/retrieval.py

@@ -19,12 +19,20 @@ class RetrievalSectionExclusion(TypedDict):
     section_path: str
 
 
+class TimeRange(BaseModel):
+    """Audio/video time span in milliseconds from the start of the recording."""
+
+    start_ms: int
+    end_ms: int
+
+
 class RetrievalSource(BaseModel):
     """Caller-facing source reference attached to a retrieval result."""
 
     document_id: Optional[str] = None
     source_file_name: Optional[str] = None
     section_path: Optional[str] = None
+    time_ranges: Optional[list[TimeRange]] = None
 
 
 class RetrievalTextPart(BaseModel):
@@ -71,6 +79,7 @@ class RetrievalReferencedChunk(BaseModel):
     file_path: Optional[str] = None
     job_id: Optional[str] = None
     asset_url: Optional[str] = None
+    time_ranges: Optional[list[TimeRange]] = None
     metadata: Optional[dict[str, Any]] = None
 
 

tests/test_retrieval.py

@@ -368,3 +368,26 @@ def test_legacy_response_without_agentic_fields(self, sync_client: Any) -> None:
         assert not hasattr(response.results[0], "composed")
         assert response.referenced_chunks == []
         assert response.decision_trace is None
+
+
+def test_media_citations_keep_time_ranges() -> None:
+    from knowhere.types.retrieval import RetrievalQueryResponse, TimeRange
+
+    payload = _make_retrieval_response()
+    span = {"start_ms": 723_000, "end_ms": 860_000}
+    payload["results"][0]["source"]["time_ranges"] = [span]
+    payload["referenced_chunks"][0]["time_ranges"] = [span]
+
+    response = RetrievalQueryResponse.model_validate(payload)
+
+    assert response.results[0].source.time_ranges == [TimeRange(**span)]
+    assert response.referenced_chunks[0].time_ranges == [TimeRange(**span)]
+
+
+def test_document_citations_have_no_time_ranges() -> None:
+    from knowhere.types.retrieval import RetrievalQueryResponse
+
+    response = RetrievalQueryResponse.model_validate(_make_retrieval_response())
+
+    assert response.results[0].source.time_ranges is None
+    assert response.referenced_chunks[0].time_ranges is None