diff --git a/eval/latest.json b/eval/latest.json new file mode 100644 index 0000000..0ce7896 --- /dev/null +++ b/eval/latest.json @@ -0,0 +1,891 @@ +{ + "api": "https://floatchat-ai-svjw.onrender.com", + "ran_at": "2026-11-08 18:16 UTC", + "summary": { + "cases": 30, + "passed": 29, + "routing_accuracy": 0.967, + "filter_accuracy": 1.0, + "answer_pass_rate": 1.0, + "median_latency_ms": 2595, + "llm_routed": 28, + "llm_wrote_answer": 30, + "failed_requests": 0 + }, + "results": [ + { + "id": "depth-avg", + "question": "What is the average temperature at different depths?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "avg_by_depth" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "avg_by_depth", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2701, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "depth-avg-arabian", + "question": "What is the average temperature at different depths in the Arabian Sea?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "avg_by_depth" + ], + "filters": { + "region": "Arabian Sea" + } + }, + "got": { + "path": "analytical", + "intent": "avg_by_depth", + "filters": { + "region": "Arabian Sea" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1498, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "depth-avg-bengal-march", + "question": "Average salinity by depth in the Bay of Bengal in March 2026", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "avg_by_depth" + ], + "filters": { + "region": "Bay of Bengal", + "month_num": 3, + "year": 2026 + } + }, + "got": { + "path": "analytical", + "intent": "avg_by_depth", + "filters": { + "region": "Bay of Bengal", + "month_num": 3, + "year": 2026 + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1712, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "regions", + "question": "Compare surface temperature across ocean regions", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "regional_comparison" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "regional_comparison", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1957, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "regions-warmest", + "question": "Which region has the highest average temperature in the upper ocean?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "regional_comparison" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "regional_comparison", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1836, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "layers", + "question": "Show temperature and salinity by depth layer", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "depth_ranges" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "depth_ranges", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1807, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "layers-southern", + "question": "Compare surface and deep water in the Southern Indian Ocean", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "depth_ranges" + ], + "filters": { + "region": "Southern Indian Ocean" + } + }, + "got": { + "path": "analytical", + "intent": "depth_ranges", + "filters": { + "region": "Southern Indian Ocean" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1931, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "monthly", + "question": "Show the monthly trend of surface temperature", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "temporal_trends" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "temporal_trends", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1818, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "monthly-equator", + "question": "How has the average temperature near the equator changed over time?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "temporal_trends" + ], + "filters": { + "region": "Equatorial Indian Ocean" + } + }, + "got": { + "path": "analytical", + "intent": "temporal_trends", + "filters": { + "region": "Equatorial Indian Ocean" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2003, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "floats-active", + "question": "Give me a summary of the most active floats", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "float_summary" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "float_summary", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 3129, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "floats-arabian", + "question": "Summarize the floats operating in the Arabian Sea", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "float_summary" + ], + "filters": { + "region": "Arabian Sea" + } + }, + "got": { + "path": "analytical", + "intent": "float_summary", + "filters": { + "region": "Arabian Sea" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2463, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "bgc-oxygen", + "question": "Average dissolved oxygen by ocean zone", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "bgc_analysis" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "bgc_analysis", + "filters": { + "has_oxygen": true + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1996, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "bgc-chlorophyll-bengal", + "question": "Compare chlorophyll levels across zones in the Bay of Bengal", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "bgc_analysis" + ], + "filters": { + "region": "Bay of Bengal" + } + }, + "got": { + "path": "analytical", + "intent": "bgc_analysis", + "filters": { + "region": "Bay of Bengal", + "has_chlorophyll": true + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2165, + "router": "keywords", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "bgc-ph", + "question": "What is the average pH in each ocean zone?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "bgc_analysis" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "bgc_analysis", + "filters": { + "has_ph": true + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 1672, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "count-arabian", + "question": "How many profiles were recorded in the Arabian Sea?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "custom" + ], + "filters": { + "region": "Arabian Sea" + } + }, + "got": { + "path": "analytical", + "intent": "custom", + "filters": { + "region": "Arabian Sea" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 181, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "max-bengal-2026", + "question": "What was the maximum temperature recorded in the Bay of Bengal in 2026?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "custom" + ], + "filters": { + "region": "Bay of Bengal", + "year": 2026 + } + }, + "got": { + "path": "analytical", + "intent": "custom", + "filters": { + "region": "Bay of Bengal", + "year": 2026 + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2882, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "mean-salinity-1000m", + "question": "What is the mean salinity at 1000 m?", + "follow_up_of": null, + "expected": { + "path": "analytical", + "intent": [ + "custom", + "avg_by_depth" + ], + "filters": {} + }, + "got": { + "path": "analytical", + "intent": "avg_by_depth", + "filters": {}, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2162, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-equator-march", + "question": "Show me salinity profiles near the equator in March 2026", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Equatorial Indian Ocean", + "month_num": 3, + "year": 2026 + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Equatorial Indian Ocean", + "month_num": 3, + "year": 2026 + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 7575, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-low-oxygen", + "question": "Which floats measured low oxygen in the Arabian Sea?", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Arabian Sea", + "has_oxygen": true + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Arabian Sea", + "has_oxygen": true + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 3701, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-bengal", + "question": "Find profiles from the Bay of Bengal", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Bay of Bengal" + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Bay of Bengal" + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 2894, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-chlorophyll-southern", + "question": "Show me floats with chlorophyll sensors in the Southern Indian Ocean", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Southern Indian Ocean", + "has_chlorophyll": true + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Southern Indian Ocean", + "has_chlorophyll": true + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 10737, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-indonesia-january", + "question": "Profiles near Indonesia in January 2026", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Eastern Indian Ocean", + "month_num": 1, + "year": 2026 + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Eastern Indian Ocean", + "month_num": 1, + "year": 2026 + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 10217, + "router": "keywords", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "no-data-red-sea", + "question": "Describe recent profiles from the Red Sea", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Red Sea & Gulf of Aden" + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Red Sea & Gulf of Aden" + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 4217, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-nitrate-arabian", + "question": "Find nitrate measurements in the Arabian Sea", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Arabian Sea", + "has_nitrate": true + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Arabian Sea", + "has_nitrate": true + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 11665, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-thermocline", + "question": "Show me profiles with a strong thermocline", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": {} + }, + "got": { + "path": "semantic", + "intent": null, + "filters": {}, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 7732, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "find-bengal-september", + "question": "Profiles in the Bay of Bengal in September 2026", + "follow_up_of": null, + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Bay of Bengal", + "month_num": 9, + "year": 2026 + } + }, + "got": { + "path": "analytical", + "intent": "custom", + "filters": { + "region": "Bay of Bengal", + "month_num": 9, + "year": 2026 + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 4907, + "router": "llm", + "llm_wrote_answer": true, + "routing": false, + "filters": true, + "answer": true, + "note": "routed to analytical/custom" + }, + { + "id": "follow-region", + "question": "now only the Bay of Bengal", + "follow_up_of": "depth-avg", + "expected": { + "path": "analytical", + "intent": [ + "avg_by_depth" + ], + "filters": { + "region": "Bay of Bengal" + } + }, + "got": { + "path": "analytical", + "intent": "avg_by_depth", + "filters": { + "region": "Bay of Bengal" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 3397, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "follow-month", + "question": "and just in March", + "follow_up_of": "depth-avg-arabian", + "expected": { + "path": "analytical", + "intent": [ + "avg_by_depth" + ], + "filters": { + "region": "Arabian Sea", + "month_num": 3 + } + }, + "got": { + "path": "analytical", + "intent": "avg_by_depth", + "filters": { + "region": "Arabian Sea", + "month_num": 3 + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 5795, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "follow-monthly-arabian", + "question": "now for the Arabian Sea", + "follow_up_of": "monthly", + "expected": { + "path": "analytical", + "intent": [ + "temporal_trends" + ], + "filters": { + "region": "Arabian Sea" + } + }, + "got": { + "path": "analytical", + "intent": "temporal_trends", + "filters": { + "region": "Arabian Sea" + }, + "answered_by": "analytical" + }, + "status": 200, + "latency_ms": 2489, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + }, + { + "id": "follow-oxygen", + "question": "only the ones measuring oxygen", + "follow_up_of": "find-bengal", + "expected": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Bay of Bengal", + "has_oxygen": true + } + }, + "got": { + "path": "semantic", + "intent": null, + "filters": { + "region": "Bay of Bengal", + "has_oxygen": true + }, + "answered_by": "semantic" + }, + "status": 200, + "latency_ms": 13054, + "router": "llm", + "llm_wrote_answer": true, + "routing": true, + "filters": true, + "answer": true, + "note": null + } + ] +} diff --git a/eval/latest.md b/eval/latest.md new file mode 100644 index 0000000..8ae8df8 --- /dev/null +++ b/eval/latest.md @@ -0,0 +1,45 @@ +## FloatChat evaluation — 29/30 questions fully correct + +Run against `https://floatchat-ai-svjw.onrender.com` on 2026-11-08 18:16 UTC. + +| Metric | Score | +|---|---| +| Routing accuracy (SQL vs search, and which analysis) | 97% | +| Filter accuracy (region, month, year, sensor) | 100% | +| Answer checks passed | 100% | +| Median latency | 2595 ms | +| Understood by the LLM router | 28/30 | +| Answers written by the LLM | 30/30 | + +| Case | Question | Routing | Filters | Answer | Notes | +|---|---|---|---|---|---| +| `depth-avg` | What is the average temperature at different depths? | ✅ | ✅ | ✅ | | +| `depth-avg-arabian` | What is the average temperature at different depths in the Arabian Sea? | ✅ | ✅ | ✅ | | +| `depth-avg-bengal-march` | Average salinity by depth in the Bay of Bengal in March 2026 | ✅ | ✅ | ✅ | | +| `regions` | Compare surface temperature across ocean regions | ✅ | ✅ | ✅ | | +| `regions-warmest` | Which region has the highest average temperature in the upper ocean? | ✅ | ✅ | ✅ | | +| `layers` | Show temperature and salinity by depth layer | ✅ | ✅ | ✅ | | +| `layers-southern` | Compare surface and deep water in the Southern Indian Ocean | ✅ | ✅ | ✅ | | +| `monthly` | Show the monthly trend of surface temperature | ✅ | ✅ | ✅ | | +| `monthly-equator` | How has the average temperature near the equator changed over time? | ✅ | ✅ | ✅ | | +| `floats-active` | Give me a summary of the most active floats | ✅ | ✅ | ✅ | | +| `floats-arabian` | Summarize the floats operating in the Arabian Sea | ✅ | ✅ | ✅ | | +| `bgc-oxygen` | Average dissolved oxygen by ocean zone | ✅ | ✅ | ✅ | | +| `bgc-chlorophyll-bengal` | Compare chlorophyll levels across zones in the Bay of Bengal | ✅ | ✅ | ✅ | | +| `bgc-ph` | What is the average pH in each ocean zone? | ✅ | ✅ | ✅ | | +| `count-arabian` | How many profiles were recorded in the Arabian Sea? | ✅ | ✅ | ✅ | | +| `max-bengal-2026` | What was the maximum temperature recorded in the Bay of Bengal in 2026? | ✅ | ✅ | ✅ | | +| `mean-salinity-1000m` | What is the mean salinity at 1000 m? | ✅ | ✅ | ✅ | | +| `find-equator-march` | Show me salinity profiles near the equator in March 2026 | ✅ | ✅ | ✅ | | +| `find-low-oxygen` | Which floats measured low oxygen in the Arabian Sea? | ✅ | ✅ | ✅ | | +| `find-bengal` | Find profiles from the Bay of Bengal | ✅ | ✅ | ✅ | | +| `find-chlorophyll-southern` | Show me floats with chlorophyll sensors in the Southern Indian Ocean | ✅ | ✅ | ✅ | | +| `find-indonesia-january` | Profiles near Indonesia in January 2026 | ✅ | ✅ | ✅ | | +| `no-data-red-sea` | Describe recent profiles from the Red Sea | ✅ | ✅ | ✅ | | +| `find-nitrate-arabian` | Find nitrate measurements in the Arabian Sea | ✅ | ✅ | ✅ | | +| `find-thermocline` | Show me profiles with a strong thermocline | ✅ | ✅ | ✅ | | +| `find-bengal-september` | Profiles in the Bay of Bengal in September 2026 | ❌ | ✅ | ✅ | routed to analytical/custom | +| `follow-region` | now only the Bay of Bengal *(follow-up to depth-avg)* | ✅ | ✅ | ✅ | | +| `follow-month` | and just in March *(follow-up to depth-avg-arabian)* | ✅ | ✅ | ✅ | | +| `follow-monthly-arabian` | now for the Arabian Sea *(follow-up to monthly)* | ✅ | ✅ | ✅ | | +| `follow-oxygen` | only the ones measuring oxygen *(follow-up to find-bengal)* | ✅ | ✅ | ✅ | | diff --git a/readme.md b/readme.md index c786133..10118fd 100644 --- a/readme.md +++ b/readme.md @@ -197,7 +197,14 @@ every Monday (`eval.yml`) and shows the table in the job summary. Without an LLM template questions and simple follow-ups; the free-phrasing and custom-SQL questions are where the LLM earns its keep. -**Latest score:** see [`eval/latest.md`](eval/latest.md). +**Latest score (live API, 8 Nov 2026): 29/30 questions fully correct.** + +| Routing | Filters | Answer checks | Median latency | Answers written by the LLM | +|---|---|---|---|---| +| 97% | 100% | 100% | 2.6 s | 30/30 | + +The one miss: *"Profiles in the Bay of Bengal in September 2026"* went to custom SQL instead of profile search. The +full per-question table is in [`eval/latest.md`](eval/latest.md). ---