Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ between local, self-hosted, and cloud is a one-variable change.
```python
PromptClient(host=None, token=None, tls=False, ca_cert=None,
client_cert=None, client_key=None,
cache_ttl=0, nats_url=None, url=None)
cache_ttl=0, cache_max_entries=1024, nats_url=None, url=None)
```

| Parameter | What it does |
Expand All @@ -70,6 +70,7 @@ PromptClient(host=None, token=None, tls=False, ca_cert=None,
| `token` | sent as `authorization: Bearer <token>`; defaults to `PRIOMPT_TOKEN`, then the url's token |
| `tls` / `ca_cert` | use TLS, optionally pinning a CA certificate |
| `cache_ttl` | local cache lifetime in seconds (`0` = off) — repeat `get`s within the TTL never touch the network |
| `cache_max_entries` | upper bound on cached entries (default 1024); the least recently used entry is evicted when full |
| `nats_url` | endpoint for `subscribe()` |

## Everything the client can do
Expand Down
55 changes: 45 additions & 10 deletions priompt_sdk/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,16 @@
import os
import threading
import time
from collections import OrderedDict
from urllib.parse import urlparse

import grpc

from priompt_sdk.v1 import prompt_pb2, prompt_pb2_grpc


_MISS = object() # cache lookup miss; a cached value is never this

_TOKEN_SAFE = frozenset(
b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-"
)
Expand Down Expand Up @@ -51,6 +54,7 @@ def __init__(
client_cert=None,
client_key=None,
cache_ttl=0,
cache_max_entries=1024,
nats_url=None,
url=None,
):
Expand Down Expand Up @@ -94,9 +98,39 @@ def __init__(
# collide. The server only returns validated prompts, so anything cached is
# valid; TTL is the staleness bound (a change shows up within cache_ttl s).
self._cache_ttl = cache_ttl
self._cache = {}
# Bounded LRU: expired entries are dropped when looked up, and the
# oldest is evicted when the cache is full. Ops go through _cache_lock
# because a lookup is read-modify-write (delete expired + move_to_end)
# and subscribe()'s handler touches the cache from its own thread.
self._cache_max_entries = max(1, cache_max_entries)
self._cache = OrderedDict()
self._cache_lock = threading.Lock()
self._nats_url = nats_url # e.g. "nats://your-company.prompts.io:4222"

def _cache_lookup(self, key):
with self._cache_lock:
hit = self._cache.get(key, _MISS)
if hit is _MISS:
return _MISS
if hit[1] <= time.monotonic():
del self._cache[key]
return _MISS
self._cache.move_to_end(key)
return hit[0]

def _cache_store(self, key, value):
with self._cache_lock:
now = time.monotonic()
# Sweep expired entries on insert, not only on lookup: keys that
# are never asked for again would otherwise accumulate forever.
expired = [k for k, (_, exp) in self._cache.items() if exp <= now]
for k in expired:
del self._cache[k]
self._cache[key] = (value, now + self._cache_ttl)
self._cache.move_to_end(key)
while len(self._cache) > self._cache_max_entries:
self._cache.popitem(last=False)

def get(self, uri, ref=""):
"""Fetch a prompt by priompt:// URI. Returns the GetPromptResponse.

Expand All @@ -108,14 +142,14 @@ def get(self, uri, ref=""):
prompt_pb2.GetPromptRequest(uri=uri, ref=ref), metadata=self._md
)
if self._cache_ttl:
hit = self._cache.get(uri)
if hit and hit[1] > time.monotonic():
return hit[0]
hit = self._cache_lookup(uri)
if hit is not _MISS:
return hit
resp = self._stub.GetPrompt(
prompt_pb2.GetPromptRequest(uri=uri), metadata=self._md
)
if self._cache_ttl:
self._cache[uri] = (resp, time.monotonic() + self._cache_ttl)
self._cache_store(uri, resp)
return resp

def list(self, prefix=""):
Expand All @@ -129,14 +163,14 @@ def list(self, prefix=""):
"""
key = ("list", prefix)
if self._cache_ttl:
hit = self._cache.get(key)
if hit and hit[1] > time.monotonic():
return hit[0]
hit = self._cache_lookup(key)
if hit is not _MISS:
return hit
entries = self._stub.ListPrompts(
prompt_pb2.ListPromptsRequest(prefix=prefix), metadata=self._md
).entries
if self._cache_ttl:
self._cache[key] = (entries, time.monotonic() + self._cache_ttl)
self._cache_store(key, entries)
return entries

def diff(self, uri, new_template):
Expand Down Expand Up @@ -213,7 +247,8 @@ async def handler(msg):
# Only an accepted change refreshes the cache; a held one
# keeps serving the cached version until cache_ttl expires.
if accept and self._cache_ttl:
self._cache.pop(uri, None) # drop stale cache
with self._cache_lock:
self._cache.pop(uri, None) # drop stale cache
self.get(uri) # refetch now so the next get() is warm
on_change(version, classification)

Expand Down
33 changes: 29 additions & 4 deletions tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,33 @@ def test_list_caches_within_ttl(clock):
assert c._stub.list_calls == 1


def test_expired_entries_are_removed_for_get_and_list(clock):
"""Expired keys that are never queried again must still leave the cache:
lazy deletion alone leaves distinct-URI users growing without bound."""
c = make_client(cache_ttl=10)
for i in range(50):
c.get(f"priompt://p{i}")
c.list(f"team{i}/")
clock.advance(11)
c.get("priompt://fresh")
assert len(c._cache) == 1 # both get() and list() keys were swept
assert "priompt://fresh" in c._cache


def test_cache_is_bounded_and_evicts_least_recently_used(clock):
c = PromptClient("localhost:1", cache_ttl=10, cache_max_entries=2)
c._stub = FakeStub()
c.get("priompt://a")
c.get("priompt://b")
c.get("priompt://a") # cache hit: a becomes most recently used
c.get("priompt://c") # evicts b, the least recently used
assert len(c._cache) == 2
assert "priompt://a" in c._cache and "priompt://b" not in c._cache
for i in range(10):
c.get(f"priompt://k{i}")
assert len(c._cache) == 2 # never grows past the bound


def test_diff_is_never_cached():
c = make_client(cache_ttl=10)
c.diff("priompt://x", "new template")
Expand Down Expand Up @@ -260,10 +287,8 @@ def test_reload_policy_can_be_a_function(clock, fake_nats):
# --- stress: lock-free cache under concurrent readers ---------------------

def test_concurrent_gets_do_not_crash_or_corrupt(clock):
"""The cache is a plain dict with no lock (ponytail: fine for CPython, RPC
dedup under a race is best-effort). Hammer it from many threads on many
keys and assert every reader got a correct, self-consistent value and the
dict didn't corrupt."""
"""Hammer the shared cache from many threads on many keys and assert every
reader got a correct, self-consistent value and the cache stayed intact."""
c = make_client(cache_ttl=100)
keys = [f"priompt://k{i}" for i in range(20)]
errors = []
Expand Down
Loading