From f99c5525c336bc152cad435d0d7233ba66e8a3c9 Mon Sep 17 00:00:00 2001 From: bfs2021 <101915083+bfs2021@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:11:15 +0800 Subject: [PATCH] Bound the client cache and drop expired entries The cache only ever grew: expired entries were skipped on lookup but never removed, and nothing capped the number of keys, so a long-lived process touching many URIs or list() prefixes held every response for its life. Prune expired entries on store (covering both get() and list() keys) and keep the cache in an OrderedDict as a bounded LRU (cache_max_entries, default 1024). Cache operations take a lock because lookups are now read-modify-write and subscribe()'s handler touches the cache from its own thread. Signed-off-by: bfs2021 <101915083+bfs2021@users.noreply.github.com> --- README.md | 3 ++- priompt_sdk/client.py | 55 +++++++++++++++++++++++++++++++++++-------- tests/test_client.py | 33 ++++++++++++++++++++++---- 3 files changed, 76 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b37fce4..b0ea513 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ between local, self-hosted, and cloud is a one-variable change. ```python PromptClient(host=None, token=None, tls=False, ca_cert=None, client_cert=None, client_key=None, - cache_ttl=0, nats_url=None, url=None) + cache_ttl=0, cache_max_entries=1024, nats_url=None, url=None) ``` | Parameter | What it does | @@ -70,6 +70,7 @@ PromptClient(host=None, token=None, tls=False, ca_cert=None, | `token` | sent as `authorization: Bearer `; defaults to `PRIOMPT_TOKEN`, then the url's token | | `tls` / `ca_cert` | use TLS, optionally pinning a CA certificate | | `cache_ttl` | local cache lifetime in seconds (`0` = off) — repeat `get`s within the TTL never touch the network | +| `cache_max_entries` | upper bound on cached entries (default 1024); the least recently used entry is evicted when full | | `nats_url` | endpoint for `subscribe()` | ## Everything the client can do diff --git a/priompt_sdk/client.py b/priompt_sdk/client.py index 086c58f..db6c6dd 100644 --- a/priompt_sdk/client.py +++ b/priompt_sdk/client.py @@ -4,6 +4,7 @@ import os import threading import time +from collections import OrderedDict from urllib.parse import urlparse import grpc @@ -11,6 +12,8 @@ from priompt_sdk.v1 import prompt_pb2, prompt_pb2_grpc +_MISS = object() # cache lookup miss; a cached value is never this + _TOKEN_SAFE = frozenset( b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" ) @@ -51,6 +54,7 @@ def __init__( client_cert=None, client_key=None, cache_ttl=0, + cache_max_entries=1024, nats_url=None, url=None, ): @@ -94,9 +98,39 @@ def __init__( # collide. The server only returns validated prompts, so anything cached is # valid; TTL is the staleness bound (a change shows up within cache_ttl s). self._cache_ttl = cache_ttl - self._cache = {} + # Bounded LRU: expired entries are dropped when looked up, and the + # oldest is evicted when the cache is full. Ops go through _cache_lock + # because a lookup is read-modify-write (delete expired + move_to_end) + # and subscribe()'s handler touches the cache from its own thread. + self._cache_max_entries = max(1, cache_max_entries) + self._cache = OrderedDict() + self._cache_lock = threading.Lock() self._nats_url = nats_url # e.g. "nats://your-company.prompts.io:4222" + def _cache_lookup(self, key): + with self._cache_lock: + hit = self._cache.get(key, _MISS) + if hit is _MISS: + return _MISS + if hit[1] <= time.monotonic(): + del self._cache[key] + return _MISS + self._cache.move_to_end(key) + return hit[0] + + def _cache_store(self, key, value): + with self._cache_lock: + now = time.monotonic() + # Sweep expired entries on insert, not only on lookup: keys that + # are never asked for again would otherwise accumulate forever. + expired = [k for k, (_, exp) in self._cache.items() if exp <= now] + for k in expired: + del self._cache[k] + self._cache[key] = (value, now + self._cache_ttl) + self._cache.move_to_end(key) + while len(self._cache) > self._cache_max_entries: + self._cache.popitem(last=False) + def get(self, uri, ref=""): """Fetch a prompt by priompt:// URI. Returns the GetPromptResponse. @@ -108,14 +142,14 @@ def get(self, uri, ref=""): prompt_pb2.GetPromptRequest(uri=uri, ref=ref), metadata=self._md ) if self._cache_ttl: - hit = self._cache.get(uri) - if hit and hit[1] > time.monotonic(): - return hit[0] + hit = self._cache_lookup(uri) + if hit is not _MISS: + return hit resp = self._stub.GetPrompt( prompt_pb2.GetPromptRequest(uri=uri), metadata=self._md ) if self._cache_ttl: - self._cache[uri] = (resp, time.monotonic() + self._cache_ttl) + self._cache_store(uri, resp) return resp def list(self, prefix=""): @@ -129,14 +163,14 @@ def list(self, prefix=""): """ key = ("list", prefix) if self._cache_ttl: - hit = self._cache.get(key) - if hit and hit[1] > time.monotonic(): - return hit[0] + hit = self._cache_lookup(key) + if hit is not _MISS: + return hit entries = self._stub.ListPrompts( prompt_pb2.ListPromptsRequest(prefix=prefix), metadata=self._md ).entries if self._cache_ttl: - self._cache[key] = (entries, time.monotonic() + self._cache_ttl) + self._cache_store(key, entries) return entries def diff(self, uri, new_template): @@ -213,7 +247,8 @@ async def handler(msg): # Only an accepted change refreshes the cache; a held one # keeps serving the cached version until cache_ttl expires. if accept and self._cache_ttl: - self._cache.pop(uri, None) # drop stale cache + with self._cache_lock: + self._cache.pop(uri, None) # drop stale cache self.get(uri) # refetch now so the next get() is warm on_change(version, classification) diff --git a/tests/test_client.py b/tests/test_client.py index 4d0c799..1e25ed9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -158,6 +158,33 @@ def test_list_caches_within_ttl(clock): assert c._stub.list_calls == 1 +def test_expired_entries_are_removed_for_get_and_list(clock): + """Expired keys that are never queried again must still leave the cache: + lazy deletion alone leaves distinct-URI users growing without bound.""" + c = make_client(cache_ttl=10) + for i in range(50): + c.get(f"priompt://p{i}") + c.list(f"team{i}/") + clock.advance(11) + c.get("priompt://fresh") + assert len(c._cache) == 1 # both get() and list() keys were swept + assert "priompt://fresh" in c._cache + + +def test_cache_is_bounded_and_evicts_least_recently_used(clock): + c = PromptClient("localhost:1", cache_ttl=10, cache_max_entries=2) + c._stub = FakeStub() + c.get("priompt://a") + c.get("priompt://b") + c.get("priompt://a") # cache hit: a becomes most recently used + c.get("priompt://c") # evicts b, the least recently used + assert len(c._cache) == 2 + assert "priompt://a" in c._cache and "priompt://b" not in c._cache + for i in range(10): + c.get(f"priompt://k{i}") + assert len(c._cache) == 2 # never grows past the bound + + def test_diff_is_never_cached(): c = make_client(cache_ttl=10) c.diff("priompt://x", "new template") @@ -260,10 +287,8 @@ def test_reload_policy_can_be_a_function(clock, fake_nats): # --- stress: lock-free cache under concurrent readers --------------------- def test_concurrent_gets_do_not_crash_or_corrupt(clock): - """The cache is a plain dict with no lock (ponytail: fine for CPython, RPC - dedup under a race is best-effort). Hammer it from many threads on many - keys and assert every reader got a correct, self-consistent value and the - dict didn't corrupt.""" + """Hammer the shared cache from many threads on many keys and assert every + reader got a correct, self-consistent value and the cache stayed intact.""" c = make_client(cache_ttl=100) keys = [f"priompt://k{i}" for i in range(20)] errors = []