Working inside the window

def audit(messages, system, tools, count_tokens):
    """Where the window is actually going, largest first."""
    rows = [
        ("system", count_tokens(system)),
        ("tool definitions", count_tokens(str(tools))),
    ]
    by_role = {}
    for m in messages:
        by_role[m["role"]] = by_role.get(m["role"], 0) + count_tokens(str(m["content"]))
    rows += sorted(by_role.items(), key=lambda kv: -kv[1])

    total = sum(n for _, n in rows)
    for label, n in sorted(rows, key=lambda kv: -kv[1]):
        print(f"{label:20} {n:7,}  {n / total:5.1%}")
    return total

# system                 420   1.2%
# tool                22,100  63.4%   <- the whole problem
# user                 6,300  18.1%
# assistant            5,030  14.4%

def trim(messages, count_tokens, budget, keep_recent=6):
    """Keep the most recent exchanges that fit, without orphaning tool results."""
    kept, used = [], 0
    for m in reversed(messages):
        size = count_tokens(str(m["content"]))
        if len(kept) >= keep_recent and used + size > budget:
            break
        kept.append(m)
        used += size
    kept.reverse()

    # A tool result whose tool_use is gone will be rejected by the API.
    call_ids = {
        c["id"]
        for m in kept if isinstance(m["content"], list)
        for c in m["content"] if c.get("type") == "tool_use"
    }
    kept = [
        m for m in kept
        if not (isinstance(m["content"], list)
                and any(c.get("type") == "tool_result"
                        and c.get("tool_use_id") not in call_ids
                        for c in m["content"]))
    ]

    # A conversation may not begin on an assistant turn.
    while kept and kept[0]["role"] != "user":
        kept.pop(0)
    return kept

SUMMARISE = """Summarise the conversation below for an assistant that will
continue it and cannot see the original. Record only:
- decisions made and constraints agreed, verbatim where exact wording matters
- facts established, with the identifiers, names and numbers involved
- what has been tried and failed, so it is not tried again
- what the user is currently trying to do

Omit pleasantries, restated questions and anything already superseded.
Write it as notes, not prose."""

def compact(client, messages, count_tokens, budget, keep_recent=6):
    if count_tokens(str(messages)) <= budget:
        return messages

    old, recent = messages[:-keep_recent], messages[-keep_recent:]
    if not old:
        return messages

    summary = client.messages.create(
        model="claude-sonnet-5",
        max_tokens=1024,
        system=SUMMARISE,
        messages=[{"role": "user", "content": str(old)}],
    ).content[0].text

    return [
        {"role": "user", "content": f"Summary of the conversation so far:
{summary}"},
        {"role": "assistant", "content": "Understood, I have that context."},
        *recent,
    ]

STORE = {}

def offload(result, keep=200):
    """Replace a large tool result with a stub the model can expand."""
    text = str(result)
    if len(text) <= keep * 2:
        return text
    ref = f"res_{len(STORE)}"
    STORE[ref] = text
    return (
        f"{text[:keep]}
"
        f"[truncated, {len(text):,} chars total. "
        f"Call read_result(ref={ref!r}, start=..., length=...) for the rest.]"
    )

def read_result(ref, start=0, length=2000):
    """Expose this as a tool so the model can pull back what it needs."""
    return STORE[ref][start : start + length]

system prompt          never changes        cached
tool definitions       never changes        cached
summary of old turns   changes rarely       cached until it does
recent exchanges       changes every turn   not cached
current question       changes every turn   not cached