ai-agents-token-usage-tracking-openaicallbackhandler-does-not-track-token-usage-w-d771c1d3

THEN Implement a custom callback handler that extends BaseCallbackHandler. Override on_llm_start to capture model name and on_llm_end to accumulate token counts (total_tokens, prompt_tokens, completion_tokens, successful_requests). Use tiktoken to estimate tokens if needed. Attach this handler to both the LLM and the AgentExecutor.

class TokenTracker(BaseCallbackHandler):
    total_tokens = 0
    prompt_tokens = 0
    completion_tokens = 0
    successful_requests = 0
    total_cost = 0.0
    model_name = ''

    def on_llm_start(self, serialized, prompts, **kwargs):
        self.model_name = serialized['kwargs']['model_name']

    def on_llm_end(self, response, **kwargs):
        if response.llm_output:
            usage = response.llm_output.get('token_usage', {})
            self.total_tokens += usage.get('total_tokens', 0)
            self.prompt_tokens += usage.get('prompt_tokens', 0)
            self.completion_tokens += usage.get('completion_tokens', 0)
            self.successful_requests += 1

cb = TokenTracker()
llm = ChatOpenAI(model='gpt-4', callbacks=[cb])
agent = AgentExecutor(agent=agent, tools=tools, callbacks=[cb])
agent.invoke({'input': '...'})
print(cb.total_tokens)