The cost maths, and caching whole responses

Illustrative prices per million input tokens (check your provider):
  normal input   $3.00
  cache write    $3.75
  cache read     $0.30

Prefix of 8,000 tokens, 1,000 requests in the cache lifetime:
  without cache  1,000 x 8,000 x $3.00 / 1M          = $24.00
  with cache     1 write:  8,000 x $3.75 / 1M        = $0.03
                 999 reads: 999 x 8,000 x $0.30 / 1M = $2.40
                                                total = about $2.43

import math

BASE, WRITE, READ = 1.0, 1.25, 0.10      # multipliers on a normal input token
TTL = 300                                # seconds an unused entry survives

def effective(gap, ttl=TTL):
    """Poisson arrivals: the chance the previous request is still cached."""
    p_hit = 1 - math.exp(-ttl / gap)
    return p_hit, p_hit * READ + (1 - p_hit) * WRITE

print(f"{'mean gap':>10}{'hit rate':>10}{'cost/req':>10}{'vs no cache':>13}")
for gap in (5, 30, 120, 300, 900, 1800, 3600):
    p, c = effective(gap)
    print(f"{gap:>9}s{p:>10.0%}{c:>10.2f}{1 - c / BASE:>12.0%}")

p_break = (WRITE - BASE) / (WRITE - READ)
gap_break = -TTL / math.log(1 - p_break)
print(f"break-even hit rate {p_break:.1%}, reached at a mean gap of {gap_break:.0f}s")

#   mean gap  hit rate  cost/req  vs no cache
#         5s      100%      0.10         90%
#        30s      100%      0.10         90%
#       120s       92%      0.19         81%
#       300s       63%      0.52         48%
#       900s       28%      0.92          8%
#      1800s       15%      1.07         -7%
#      3600s        8%      1.16        -16%
# break-even hit rate 21.7%, reached at a mean gap of 1224s

import hashlib, json

def cache_key(model, prompt_version, messages, settings, tenant_id):
    raw = json.dumps([model, prompt_version, messages, settings, tenant_id], sort_keys=True)
    return hashlib.sha256(raw.encode()).hexdigest()

a = {"temperature": 0.0, "max_tokens": 512}
b = {"max_tokens": 512, "temperature": 0.0}      # the same settings, built elsewhere
msgs = [{"role": "user", "content": "What is the refund window?"}]

print("same key with sort_keys:  ",
      cache_key("m-1", "v3", msgs, a, "acme")[:12],
      cache_key("m-1", "v3", msgs, b, "acme")[:12])
print("without sort_keys:        ",
      hashlib.sha256(json.dumps(a).encode()).hexdigest()[:12],
      hashlib.sha256(json.dumps(b).encode()).hexdigest()[:12])
print("different tenant:         ",
      cache_key("m-1", "v3", msgs, a, "acme")[:12],
      cache_key("m-1", "v3", msgs, a, "globex")[:12])

# same key with sort_keys:   5d721c1d80c1 5d721c1d80c1
# without sort_keys:         0f6fb34da972 a78cfe8d527c
# different tenant:          5d721c1d80c1 b19d2c710f32

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity

CACHED = {
    "How do I cancel my subscription?": "Open Billing and choose Cancel plan.",
    "What is the refund window on the monthly plan?": "Fourteen days, in full.",
    "How do I reset my password?": "Use Forgot password on the sign-in page.",
    "Is there a discount for annual billing?": "Yes, two months free.",
    "What is the rate limit on the free tier?": "Sixty requests per minute.",
}
PARAPHRASE = {
    "How can I cancel my subscription?": "How do I cancel my subscription?",
    "Where do I reset my password?": "How do I reset my password?",
    "What is the refund period for monthly plans?": "What is the refund window on the monthly plan?",
    "Do you give a discount if I pay annually?": "Is there a discount for annual billing?",
    "How many requests per minute does the free tier allow?": "What is the rate limit on the free tier?",
}
NEAR_MISS = [
    "What is the refund window on the annual plan?",
    "What is the rate limit on the paid tier?",
    "I do not want to cancel my subscription, how do I keep it?",
    "How do I reset another user's password?",
    "Is there a discount for annual billing for non-profits?",
    "How do I cancel my free trial before it becomes a subscription?",
]

keys = list(CACHED)
vec = TfidfVectorizer().fit(keys + list(PARAPHRASE) + NEAR_MISS)   # stands in for embeddings
K = vec.transform(keys)

def nearest(question):
    sims = cosine_similarity(vec.transform([question]), K)[0]
    i = int(sims.argmax())
    return keys[i], float(sims[i])

print(f"{'threshold':>10}{'true hits':>11}{'wrong hits':>12}{'misses':>8}")
for t in [0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90]:
    hits = sum(nearest(q)[1] >= t and nearest(q)[0] == want for q, want in PARAPHRASE.items())
    wrong = sum(nearest(q)[1] >= t for q in NEAR_MISS)
    print(f"{t:>10.2f}{hits:>11}{wrong:>12}{len(PARAPHRASE) - hits:>8}")

right = [nearest(q)[1] for q, want in PARAPHRASE.items() if nearest(q)[0] == want]
wrong = [nearest(q)[1] for q in NEAR_MISS]
print(f"lowest correct paraphrase {min(right):.2f}, highest near-miss {max(wrong):.2f}")

for q in sorted(NEAR_MISS, key=lambda q: -nearest(q)[1])[:3]:
    match, sim = nearest(q)
    print(f"{sim:.2f}  {q}")
    print(f"      served: {CACHED[match]}")

#  threshold  true hits  wrong hits  misses
#       0.30          3           6       2
#       0.40          3           6       2
#       0.50          3           6       2
#       0.60          2           4       3
#       0.70          2           3       3
#       0.80          0           3       5
#       0.90          0           0       5
# lowest correct paraphrase 0.13, highest near-miss 0.88
# 0.88  What is the refund window on the annual plan?
#       served: Fourteen days, in full.
# 0.86  What is the rate limit on the paid tier?
#       served: Sixty requests per minute.
# 0.82  Is there a discount for annual billing for non-profits?
#       served: Yes, two months free.