TODO

Once I have this, I will completely separate the argentic code, from the code that streams to the client

One function will generate a response in the agentic loop, returns a async iterable that lasts throughout all the iterations and is streamed by the api endpoint that simply streams the data streight to the user

import OpenAI from "npm:openai";
 
function initStream<T>(stream: AsyncIterable<T>, controller: AbortController){
    const chunks: T[] = [];
    let resolve: (chunk: T) => void = () => {};
    let nextChunk = new Promise<T>(r => resolve = r);
    
    controller.signal.throwIfAborted();
    controller.signal.addEventListener('abort', () => resolve(null as T));
    
    (async () => {
        for await (const chunk of stream){
            resolve(chunk);
            nextChunk = new Promise<T>(r => resolve = r);
            chunks.push(chunk);
        }
        controller.abort();
        resolve(null as T);
    })();
 
    return async function*() {
        for (const chunk of chunks) yield chunk;
        
        while(true){
            const chunk = await nextChunk;
            if(controller.signal.aborted)return;
            yield chunk;
        }
    };
}
 
const openai = new OpenAI();
const controller = new AbortController();
const stream = await openai.chat.completions.create(
    {
        stream: true,
        model: "chatgpt-4o-latest", 
        messages: [{role: "user", content: "hello"}],
        
    },
    {signal: controller.signal}
);
 
const iterable = initStream(stream, controller);
 
 
for await (const chunk of iterable()){
    console.log(chunk);
}

Design notes