diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b31635..5c0c94e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,8 @@ on: jobs: test: - runs-on: ubuntu-latest + # ubuntu-latest (26.04) no longer ships Python 3.9, which the package supports. + runs-on: ubuntu-24.04 strategy: matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] diff --git a/docs/usage.md b/docs/usage.md index f2a2444..edbdf41 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -316,6 +316,10 @@ for chunk in result.text_chunks: print(f" Summary: {summary}") ``` +Audio and video transcripts are text chunks with speaker-labelled lines +(`S1: ...`). Their metadata adds `time_ranges` (milliseconds), `speakers`, +`media` (`kind`, `duration_ms`) and `source_kind` (`"asr"`). + ### ImageChunk | Field | Type | Description | @@ -548,6 +552,7 @@ print(response.failure_reason) # no-answer reason, when returned for ref in response.referenced_chunks: print(ref.chunk_id, ref.document_id, ref.chunk_type, ref.content_source) print(ref.section_path, ref.file_path, ref.job_id, ref.asset_url, ref.metadata) + print(ref.time_ranges) # audio/video only: [TimeRange(start_ms, end_ms)] # results are raw path chunks for debug for result in response.results: @@ -584,6 +589,16 @@ result.metadata # Optional[dict] result.source.document_id result.source.source_file_name result.source.section_path +result.source.time_ranges # Optional[list[TimeRange]], audio/video only +``` + +Audio and video documents are speech transcripts. Their citations carry +`time_ranges` in milliseconds, so you can seek the recording: + +```python +for ref in response.referenced_chunks: + for span in ref.time_ranges or []: + print(f"{ref.section_path} @ {span.start_ms / 1000:.0f}s-{span.end_ms / 1000:.0f}s") ``` ### Limit or exclude documents or sections @@ -925,10 +940,12 @@ At `DEBUG` level, the SDK logs every HTTP request and response (with API keys re | Category | Extensions | |----------|-----------| | Documents | `.doc`, `.docx`, `.pdf`, `.txt` | -| Spreadsheets | `.xls`, `.xlsx`, `.csv` | -| Images | `.jpg`, `.jpeg`, `.png`, `.gif`, `.bmp`, `.tiff`, `.svg` | -| Presentations | `.ppt`, `.pptx` | -| Markdown | `.md` | +| Spreadsheets | `.xls`, `.xlsx`, `.csv`, `.tsv` | +| Images | `.jpg`, `.jpeg`, `.png`, `.gif`, `.webp`, `.bmp`, `.tif`, `.tiff`, `.heic`, `.heif` | +| Presentations | `.pptx` | +| Markdown and web | `.md`, `.html`, `.htm` | +| Audio | `.mp3`, `.wav`, `.m4a`, `.aac`, `.flac`, `.ogg`, `.opus` | +| Video (audio track only) | `.mp4`, `.mov`, `.webm`, `.mkv` | Use the `doc_type` parsing parameter to override auto-detection: diff --git a/src/knowhere/__init__.py b/src/knowhere/__init__.py index 7a766f5..27e304a 100644 --- a/src/knowhere/__init__.py +++ b/src/knowhere/__init__.py @@ -94,6 +94,7 @@ RetrievalResult, RetrievalSectionExclusion, RetrievalSource, + TimeRange, ) __all__: list[str] = [ @@ -150,6 +151,7 @@ "RetrievalReferencedChunk", "RetrievalSectionExclusion", "RetrievalSource", + "TimeRange", "RetrievalEvidencePart", "RetrievalQueryResponse", "RetrievalResult", diff --git a/src/knowhere/types/__init__.py b/src/knowhere/types/__init__.py index f0bda89..a62563d 100644 --- a/src/knowhere/types/__init__.py +++ b/src/knowhere/types/__init__.py @@ -57,6 +57,7 @@ RetrievalResult, RetrievalSectionExclusion, RetrievalSource, + TimeRange, ) __all__: list[str] = [ @@ -85,6 +86,7 @@ "RetrievalReferencedChunk", "RetrievalSectionExclusion", "RetrievalSource", + "TimeRange", "RetrievalEvidencePart", "RetrievalQueryResponse", "RetrievalResult", diff --git a/src/knowhere/types/result.py b/src/knowhere/types/result.py index 77c8adf..ee394fd 100644 --- a/src/knowhere/types/result.py +++ b/src/knowhere/types/result.py @@ -11,6 +11,7 @@ from pydantic import BaseModel, Field from knowhere._exceptions import ValidationError +from knowhere.types.retrieval import TimeRange # --------------------------------------------------------------------------- @@ -203,6 +204,11 @@ class ChunkMetadata(BaseModel): original_name: Optional[str] = None table_type: Optional[str] = None document_top_summary: Optional[str] = None + # Audio/video transcript chunks. + time_ranges: Optional[List[TimeRange]] = None + speakers: Optional[List[str]] = None + media: Optional[Dict[str, Any]] = None + source_kind: Optional[str] = None class BaseChunk(BaseModel): diff --git a/src/knowhere/types/retrieval.py b/src/knowhere/types/retrieval.py index cbc3c70..b753791 100644 --- a/src/knowhere/types/retrieval.py +++ b/src/knowhere/types/retrieval.py @@ -19,12 +19,20 @@ class RetrievalSectionExclusion(TypedDict): section_path: str +class TimeRange(BaseModel): + """Audio/video time span in milliseconds from the start of the recording.""" + + start_ms: int + end_ms: int + + class RetrievalSource(BaseModel): """Caller-facing source reference attached to a retrieval result.""" document_id: Optional[str] = None source_file_name: Optional[str] = None section_path: Optional[str] = None + time_ranges: Optional[list[TimeRange]] = None class RetrievalTextPart(BaseModel): @@ -71,6 +79,7 @@ class RetrievalReferencedChunk(BaseModel): file_path: Optional[str] = None job_id: Optional[str] = None asset_url: Optional[str] = None + time_ranges: Optional[list[TimeRange]] = None metadata: Optional[dict[str, Any]] = None diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py index 4037cf3..7890467 100644 --- a/tests/test_retrieval.py +++ b/tests/test_retrieval.py @@ -368,3 +368,26 @@ def test_legacy_response_without_agentic_fields(self, sync_client: Any) -> None: assert not hasattr(response.results[0], "composed") assert response.referenced_chunks == [] assert response.decision_trace is None + + +def test_media_citations_keep_time_ranges() -> None: + from knowhere.types.retrieval import RetrievalQueryResponse, TimeRange + + payload = _make_retrieval_response() + span = {"start_ms": 723_000, "end_ms": 860_000} + payload["results"][0]["source"]["time_ranges"] = [span] + payload["referenced_chunks"][0]["time_ranges"] = [span] + + response = RetrievalQueryResponse.model_validate(payload) + + assert response.results[0].source.time_ranges == [TimeRange(**span)] + assert response.referenced_chunks[0].time_ranges == [TimeRange(**span)] + + +def test_document_citations_have_no_time_ranges() -> None: + from knowhere.types.retrieval import RetrievalQueryResponse + + response = RetrievalQueryResponse.model_validate(_make_retrieval_response()) + + assert response.results[0].source.time_ranges is None + assert response.referenced_chunks[0].time_ranges is None