I have the following architecture:
VUE JS ----> NODEJS REST ---> PYTHON REST API (FLASK)
THe backend streams data properly.
The nodeJS part is with websockets like this:
app.post('/start-policy-session', async (req, res) => {
console.log('POST /start-policy-session endpoint hit');
// Your existing logic to handle the request and start a session
const sessionId = uuidv4(); // Generate a unique session ID
const data = req.body;
// Store the session data somewhere (in-memory store, database, etc.)
// For simplicity, we assume an in-memory store here
sessions[sessionId] = {
requestData: data,
status: 'pending' // Initial status
};
res.json({ sessionId });
});
wss.on('connection', (ws, req) => {
const sessionId = req.url.split('=')[1]; // Get sessionId from the URL
const sessionData = sessions[sessionId];
if (!sessionData) {
ws.send(JSON.stringify({ error: 'Session not found' }));
return ws.close();
}
const flaskApiUrl = process.env.FLASK_API_URL || 'http://localhost:5000/policy';
// Periodic ping to keep connection alive
const keepAliveInterval = setInterval(() => {
if (ws.readyState === WebSocket.OPEN) {
ws.ping();
}
}, 50000); // 50 seconds
// Start processing and streaming data
const startStream = async () => {
try {
const tokens = await getAllAccessTokens();
const flaskResponse = await axios({
method: 'post',
url: flaskApiUrl,
data: sessionData.requestData,
headers: {
'Authorization-KeyVault': `Bearer ${tokens.keyVaultToken}`,
'Authorization-OpenAI': `Bearer ${tokens.openAiToken}`,
'Authorization-Search': `Bearer ${tokens.searchToken}`
},
timeout: 240000,
responseType: 'stream'
});
flaskResponse.data.on('data', (chunk) => {
ws.send(chunk.toString());
});
flaskResponse.data.on('end', () => {
ws.send(JSON.stringify({ event: 'end' }));
ws.close();
});
flaskResponse.data.on('error', (err) => {
console.error('Error in Flask API response stream:', err);
ws.send(JSON.stringify({ event: 'error' }));
ws.close();
});
} catch (error) {
console.error('Error making request to Flask API:', error);
ws.send(JSON.stringify({ event: 'error' }));
ws.close();
}
};
startStream();
ws.on('close', () => {
console.log('Client disconnected');
});
ws.on('pong', () => {
console.log('Received pong from client');
});
});
and in our frontend (VUEJS) we consume the NODEJS WS connection.
const createWebSocket = (url) => {
this.socket = new WebSocket(url);
this.socket.onopen = () => {
console.log("websocket connection opened");
};
this.socket.onmessage = (event) => {
const chunk = event.data;
console.log('raw event data ' + event.data);
};
this.socket.onerror = (error) => {
console.error("Error with WS connection:", error);
};
this.socket.onclose = () => {
console.log("WebSocket connection closed, attempting to reconnect...");
};
return this.socket;
};
// Establish WebSocket connection
this.socket = createWebSocket(`${streamPath}?sessionId=${sessionId}`);
the code works perfectly fine, until the LLM answers take more than 60 seconds, the brwoser seems to close the connection and the .onclose event is fired, and the rest of the streaming coming from FLASK is not streamed anymore