diff --git a/README.md b/README.md index b37fce4..b0ea513 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ between local, self-hosted, and cloud is a one-variable change. ```python PromptClient(host=None, token=None, tls=False, ca_cert=None, client_cert=None, client_key=None, - cache_ttl=0, nats_url=None, url=None) + cache_ttl=0, cache_max_entries=1024, nats_url=None, url=None) ``` | Parameter | What it does | @@ -70,6 +70,7 @@ PromptClient(host=None, token=None, tls=False, ca_cert=None, | `token` | sent as `authorization: Bearer `; defaults to `PRIOMPT_TOKEN`, then the url's token | | `tls` / `ca_cert` | use TLS, optionally pinning a CA certificate | | `cache_ttl` | local cache lifetime in seconds (`0` = off) — repeat `get`s within the TTL never touch the network | +| `cache_max_entries` | upper bound on cached entries (default 1024); the least recently used entry is evicted when full | | `nats_url` | endpoint for `subscribe()` | ## Everything the client can do diff --git a/priompt_sdk/client.py b/priompt_sdk/client.py index 086c58f..db6c6dd 100644 --- a/priompt_sdk/client.py +++ b/priompt_sdk/client.py @@ -4,6 +4,7 @@ import os import threading import time +from collections import OrderedDict from urllib.parse import urlparse import grpc @@ -11,6 +12,8 @@ from priompt_sdk.v1 import prompt_pb2, prompt_pb2_grpc +_MISS = object() # cache lookup miss; a cached value is never this + _TOKEN_SAFE = frozenset( b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" ) @@ -51,6 +54,7 @@ def __init__( client_cert=None, client_key=None, cache_ttl=0, + cache_max_entries=1024, nats_url=None, url=None, ): @@ -94,9 +98,39 @@ def __init__( # collide. The server only returns validated prompts, so anything cached is # valid; TTL is the staleness bound (a change shows up within cache_ttl s). self._cache_ttl = cache_ttl - self._cache = {} + # Bounded LRU: expired entries are dropped when looked up, and the + # oldest is evicted when the cache is full. Ops go through _cache_lock + # because a lookup is read-modify-write (delete expired + move_to_end) + # and subscribe()'s handler touches the cache from its own thread. + self._cache_max_entries = max(1, cache_max_entries) + self._cache = OrderedDict() + self._cache_lock = threading.Lock() self._nats_url = nats_url # e.g. "nats://your-company.prompts.io:4222" + def _cache_lookup(self, key): + with self._cache_lock: + hit = self._cache.get(key, _MISS) + if hit is _MISS: + return _MISS + if hit[1] <= time.monotonic(): + del self._cache[key] + return _MISS + self._cache.move_to_end(key) + return hit[0] + + def _cache_store(self, key, value): + with self._cache_lock: + now = time.monotonic() + # Sweep expired entries on insert, not only on lookup: keys that + # are never asked for again would otherwise accumulate forever. + expired = [k for k, (_, exp) in self._cache.items() if exp <= now] + for k in expired: + del self._cache[k] + self._cache[key] = (value, now + self._cache_ttl) + self._cache.move_to_end(key) + while len(self._cache) > self._cache_max_entries: + self._cache.popitem(last=False) + def get(self, uri, ref=""): """Fetch a prompt by priompt:// URI. Returns the GetPromptResponse. @@ -108,14 +142,14 @@ def get(self, uri, ref=""): prompt_pb2.GetPromptRequest(uri=uri, ref=ref), metadata=self._md ) if self._cache_ttl: - hit = self._cache.get(uri) - if hit and hit[1] > time.monotonic(): - return hit[0] + hit = self._cache_lookup(uri) + if hit is not _MISS: + return hit resp = self._stub.GetPrompt( prompt_pb2.GetPromptRequest(uri=uri), metadata=self._md ) if self._cache_ttl: - self._cache[uri] = (resp, time.monotonic() + self._cache_ttl) + self._cache_store(uri, resp) return resp def list(self, prefix=""): @@ -129,14 +163,14 @@ def list(self, prefix=""): """ key = ("list", prefix) if self._cache_ttl: - hit = self._cache.get(key) - if hit and hit[1] > time.monotonic(): - return hit[0] + hit = self._cache_lookup(key) + if hit is not _MISS: + return hit entries = self._stub.ListPrompts( prompt_pb2.ListPromptsRequest(prefix=prefix), metadata=self._md ).entries if self._cache_ttl: - self._cache[key] = (entries, time.monotonic() + self._cache_ttl) + self._cache_store(key, entries) return entries def diff(self, uri, new_template): @@ -213,7 +247,8 @@ async def handler(msg): # Only an accepted change refreshes the cache; a held one # keeps serving the cached version until cache_ttl expires. if accept and self._cache_ttl: - self._cache.pop(uri, None) # drop stale cache + with self._cache_lock: + self._cache.pop(uri, None) # drop stale cache self.get(uri) # refetch now so the next get() is warm on_change(version, classification) diff --git a/tests/test_client.py b/tests/test_client.py index 4d0c799..1e25ed9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -158,6 +158,33 @@ def test_list_caches_within_ttl(clock): assert c._stub.list_calls == 1 +def test_expired_entries_are_removed_for_get_and_list(clock): + """Expired keys that are never queried again must still leave the cache: + lazy deletion alone leaves distinct-URI users growing without bound.""" + c = make_client(cache_ttl=10) + for i in range(50): + c.get(f"priompt://p{i}") + c.list(f"team{i}/") + clock.advance(11) + c.get("priompt://fresh") + assert len(c._cache) == 1 # both get() and list() keys were swept + assert "priompt://fresh" in c._cache + + +def test_cache_is_bounded_and_evicts_least_recently_used(clock): + c = PromptClient("localhost:1", cache_ttl=10, cache_max_entries=2) + c._stub = FakeStub() + c.get("priompt://a") + c.get("priompt://b") + c.get("priompt://a") # cache hit: a becomes most recently used + c.get("priompt://c") # evicts b, the least recently used + assert len(c._cache) == 2 + assert "priompt://a" in c._cache and "priompt://b" not in c._cache + for i in range(10): + c.get(f"priompt://k{i}") + assert len(c._cache) == 2 # never grows past the bound + + def test_diff_is_never_cached(): c = make_client(cache_ttl=10) c.diff("priompt://x", "new template") @@ -260,10 +287,8 @@ def test_reload_policy_can_be_a_function(clock, fake_nats): # --- stress: lock-free cache under concurrent readers --------------------- def test_concurrent_gets_do_not_crash_or_corrupt(clock): - """The cache is a plain dict with no lock (ponytail: fine for CPython, RPC - dedup under a race is best-effort). Hammer it from many threads on many - keys and assert every reader got a correct, self-consistent value and the - dict didn't corrupt.""" + """Hammer the shared cache from many threads on many keys and assert every + reader got a correct, self-consistent value and the cache stayed intact.""" c = make_client(cache_ttl=100) keys = [f"priompt://k{i}" for i in range(20)] errors = []