Handle audio too

Author: dem4ron

app/css/ui-kit/animations.css

@@ -74,3 +74,15 @@
         transform: rotate(360deg);
     }
 }
+
+@keyframes blink {
+    0% {
+        opacity: 1;
+    }
+    50% {
+        opacity: 0.3;
+    }
+    100% {
+        opacity: 1;
+    }
+}

app/javascript/components/bootcamp/FrontendExercisePage/RHS/Panels/ChatPanel/AiChat/AudioRecorder/AudioRecorder.tsx

@@ -4,6 +4,7 @@ import { useAudioRecorder } from './useAudioRecorder'
 import { assembleClassNames } from '@/utils/assemble-classnames'
 import { formatDuration } from './format-duration'
 import { GraphicalIcon } from '@/components/common'
+import { useAiChatStore } from '../store/aiChatStore'
 
 export default function AudioRecorder() {
   const {
@@ -14,6 +15,7 @@ export default function AudioRecorder() {
     startRecording,
     stopRecording,
   } = useAudioRecorder()
+  const { isResponseBeingGenerated } = useAiChatStore()
   return (
     <div
       className={assembleClassNames(
@@ -55,7 +57,11 @@ export default function AudioRecorder() {
         </button>
       ) : (
         <button
-          className="p-2 w-28 h-28 hover:bg-bootcamp-very-light-purple rounded-8"
+          className={assembleClassNames(
+            'p-2 w-28 h-28 hover:bg-bootcamp-very-light-purple rounded-8',
+            isResponseBeingGenerated && 'opacity-50'
+          )}
+          disabled={isResponseBeingGenerated}
           onClick={startRecording}
         >
           <GraphicalIcon

app/javascript/components/bootcamp/FrontendExercisePage/RHS/Panels/ChatPanel/AiChat/AudioRecorder/useAudioRecorder.ts

@@ -1,20 +1,22 @@
 import { useState, useRef, useContext, useCallback } from 'react'
-import { v4 as uuid } from 'uuid'
 import { ChatContext } from '..'
+import { convertToPCM16, resampleAudioBuffer } from '../utils'
+import { useAiChatStore } from '../store/aiChatStore'
 
 export type AudioRecorderProps = ReturnType<typeof useAudioRecorder>
 
 export function useAudioRecorder() {
   const [isRecording, setIsRecording] = useState(false)
   const [liveDuration, setLiveDuration] = useState(0)
+  const { finishStream, setIsResponseBeingGenerated } = useAiChatStore()
 
   const mediaRecorderRef = useRef<MediaRecorder | null>(null)
   const audioContextRef = useRef<AudioContext | null>(null)
   const timerRef = useRef<NodeJS.Timeout | null>(null)
   const streamRef = useRef<MediaStream | null>(null)
   const audioChunksRef = useRef<BlobPart[]>([])
 
-  const { analyserRef } = useContext(ChatContext)
+  const { analyserRef, geminiVoice } = useContext(ChatContext)
 
   const cleanup = useCallback(() => {
     if (timerRef.current) {
@@ -55,20 +57,34 @@ export function useAudioRecorder() {
     }
   }, [])
 
-  const processRecording = useCallback(() => {
+  const processRecording = useCallback(async () => {
     const chunks = audioChunksRef.current
     if (chunks.length === 0) return
 
     const mimeType = mediaRecorderRef.current?.mimeType || 'audio/webm'
     const audioBlob = new Blob(chunks, { type: mimeType })
-    const formData = new FormData()
-    formData.append('audio', audioBlob)
-
-    console.log({
-      audio: formData,
-      uuid: uuid(),
-      size: audioBlob.size,
-      duration: liveDuration,
+    const arrayBuffer = await audioBlob.arrayBuffer()
+
+    // Decode audio
+    const audioContext = new AudioContext()
+    const decoded = await audioContext.decodeAudioData(arrayBuffer)
+
+    // Downsample to 16kHz mono PCM
+    const resampledBuffer = await resampleAudioBuffer(decoded, 16000)
+    const pcmData = convertToPCM16(resampledBuffer.getChannelData(0))
+
+    // Convert to base64
+    const base64Audio = btoa(
+      String.fromCharCode(...new Uint8Array(pcmData.buffer))
+    )
+
+    console.log('Sending audio to Gemini Voice:', base64Audio)
+    geminiVoice.sendAudio(base64Audio)
+    setIsResponseBeingGenerated(true)
+    const turns = await geminiVoice.getNextTurn()
+    turns.forEach((turn) => {
+      console.log('TURNS', turn)
+      if (turn.type === 'audio') finishStream(turn.audioBlob)
     })
   }, [liveDuration])
 

app/javascript/components/bootcamp/FrontendExercisePage/RHS/Panels/ChatPanel/AiChat/ChatInput.tsx

@@ -4,15 +4,18 @@ import { assembleClassNames } from '@/utils/assemble-classnames'
 import { ChatContext } from '.'
 import { Message, useAiChatStore } from './store/aiChatStore'
 import AudioRecorder from './AudioRecorder/AudioRecorder'
-import { API_KEY, useGeminiLive } from './useGeminiLive'
 
 export function ChatInput() {
   const [value, setValue] = React.useState('')
-  const { inputRef } = useContext(ChatContext)
-  const { appendMessage, streamMessage, finishStream, isMessageBeingStreamed } =
-    useAiChatStore()
-
-  const { sendMessage, getNextTurn } = useGeminiLive(API_KEY)
+  const { inputRef, geminiText } = useContext(ChatContext)
+  const {
+    appendMessage,
+    streamMessage,
+    finishStream,
+    isMessageBeingStreamed,
+    setIsResponseBeingGenerated,
+    isResponseBeingGenerated,
+  } = useAiChatStore()
 
   const handleSendOnEnter = useCallback(
     async (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
@@ -32,17 +35,12 @@ export function ChatInput() {
         setValue('')
 
         try {
-          sendMessage(value)
-
-          const turns = await getNextTurn()
-          for (const turn of turns) {
-            console.log('turn', turn)
-            if (turn.text) {
-              streamMessage(turn.text)
-            } else if (turn.data) {
-              streamMessage(turn.data)
-            }
-          }
+          geminiText.sendText(value)
+          setIsResponseBeingGenerated(true)
+          const turns = await geminiText.getNextTurn()
+          turns.forEach((turn) => {
+            if (turn.type === 'text') streamMessage(turn.text)
+          })
 
           finishStream()
         } catch (err) {
@@ -51,7 +49,7 @@ export function ChatInput() {
         }
       }
     },
-    [value, sendMessage, getNextTurn]
+    [value, geminiText]
   )
 
   const handleInput = useCallback(
@@ -69,16 +67,23 @@ export function ChatInput() {
             ref={inputRef}
             id="text"
             name="text"
-            placeholder="Ask your question here"
+            placeholder={
+              isResponseBeingGenerated
+                ? 'Generating response...'
+                : isMessageBeingStreamed
+                ? 'Streaming response...'
+                : 'Ask anything'
+            }
             rows={1}
             className={assembleClassNames(
               'chat-textarea w-full text-16',
-              isMessageBeingStreamed && 'opacity-50'
+              isMessageBeingStreamed && 'opacity-50',
+              isResponseBeingGenerated && 'animate-blink'
             )}
             value={value}
             onInput={handleInput}
             onKeyDown={handleSendOnEnter}
-            disabled={isMessageBeingStreamed}
+            disabled={isMessageBeingStreamed || isResponseBeingGenerated}
           />
 
           {value.length > 0 && (

app/javascript/components/bootcamp/FrontendExercisePage/RHS/Panels/ChatPanel/AiChat/ChatThread.tsx

@@ -2,7 +2,6 @@ import React, { useContext, useEffect } from 'react'
 import { assembleClassNames } from '@/utils/assemble-classnames'
 import { ChatContext } from '.'
 import { useAiChatStore } from './store/aiChatStore'
-import { FAKE_LONG_STREAM_MESSAGE } from './ChatInput'
 import { useContinuousHighlighting } from '@/hooks/use-syntax-highlighting'
 
 export function ChatThread() {
@@ -27,11 +26,22 @@ export function ChatThread() {
       {messages.map((message, index) => {
         return (
           <div
-            ref={threadElementRef}
             key={message.id + index}
+            ref={threadElementRef}
             className={assembleClassNames('chat-message', message.sender)}
-            dangerouslySetInnerHTML={{ __html: message.content }}
-          />
+          >
+            {message.content && (
+              <div dangerouslySetInnerHTML={{ __html: message.content }} />
+            )}
+
+            {message.audioUrl && (
+              <audio
+                controls
+                src={message.audioUrl}
+                className="h-32 w-[300px]"
+              />
+            )}
+          </div>
         )
       })}
 

app/javascript/components/bootcamp/FrontendExercisePage/RHS/Panels/ChatPanel/AiChat/index.tsx

@@ -1,26 +1,51 @@
 import React, { createContext, useRef } from 'react'
 import { ChatInput } from './ChatInput'
 import { ChatThread } from './ChatThread'
+import { useGeminiLive } from './useGeminiLive'
+import { Modality } from '@google/genai'
+
+const API_KEY = ''
 
 type ChatContextType = {
   inputRef: React.RefObject<HTMLTextAreaElement>
   scrollContainerRef: React.RefObject<HTMLDivElement>
   analyserRef: React.MutableRefObject<AnalyserNode | null>
+  geminiText: ReturnType<typeof useGeminiLive>
+  geminiVoice: ReturnType<typeof useGeminiLive>
 }
 
 export const ChatContext = createContext<ChatContextType>({
   inputRef: {} as ChatContextType['inputRef'],
   scrollContainerRef: {} as ChatContextType['scrollContainerRef'],
   analyserRef: {} as ChatContextType['analyserRef'],
+  geminiText: {} as ChatContextType['geminiText'],
+  geminiVoice: {} as ChatContextType['geminiVoice'],
 })
 
 export function Chat() {
   const inputRef = useRef<HTMLTextAreaElement>(null)
   const scrollContainerRef = useRef<HTMLDivElement>(null)
   const analyserRef = useRef<AnalyserNode | null>(null)
 
+  const geminiText = useGeminiLive({
+    apiKey: API_KEY,
+    responseModality: Modality.TEXT,
+  })
+  const geminiVoice = useGeminiLive({
+    apiKey: API_KEY,
+    responseModality: Modality.AUDIO,
+  })
+
   return (
-    <ChatContext.Provider value={{ inputRef, scrollContainerRef, analyserRef }}>
+    <ChatContext.Provider
+      value={{
+        inputRef,
+        scrollContainerRef,
+        analyserRef,
+        geminiText,
+        geminiVoice,
+      }}
+    >
       <div className="ai-chat-container">
         <ChatThread />
         <ChatInput />

app/javascript/components/bootcamp/FrontendExercisePage/RHS/Panels/ChatPanel/AiChat/store/aiChatStore.ts

@@ -4,6 +4,7 @@ export type Message = {
   id: string
   sender: 'user' | 'ai'
   content: string
+  audioUrl?: string
   timestamp: string
 }
 
@@ -12,8 +13,10 @@ type AiChatStore = {
   appendMessage: (message: Message) => void
   messageStream: string
   streamMessage: (message: string) => void
-  finishStream: () => void
+  finishStream: (audioBlob?: Blob) => void
   isMessageBeingStreamed: boolean
+  isResponseBeingGenerated: boolean
+  setIsResponseBeingGenerated: (isGenerating: boolean) => void
 }
 
 const MOCK_MESSAGES: Message[] = [
@@ -45,23 +48,45 @@ export const useAiChatStore = create<AiChatStore>((set, get) => ({
     }))
   },
   isMessageBeingStreamed: false,
+  isResponseBeingGenerated: false,
+  setIsResponseBeingGenerated: (isGenerating: boolean) => {
+    set({ isResponseBeingGenerated: isGenerating })
+  },
   messageStream: '',
   streamMessage: (message: string) => {
     set((state) => ({
       isMessageBeingStreamed: true,
+      isResponseBeingGenerated: false,
       messageStream: state.messageStream + message,
     }))
   },
-  finishStream: () => {
+  finishStream: (audioBlob?: Blob) => {
     const { messageStream, appendMessage } = get()
-    if (messageStream.trim() === '') return
+
+    const hasText = messageStream.trim() !== ''
+    const hasAudio = !!audioBlob
+
+    if (!hasText && !hasAudio) {
+      set({
+        messageStream: '',
+        isMessageBeingStreamed: false,
+        isResponseBeingGenerated: false,
+      })
+      return
+    }
+
     const newMessage: Message = {
       id: Date.now().toString(),
       sender: 'ai',
-      content: messageStream,
+      content: hasText ? messageStream : '',
+      audioUrl: hasAudio ? URL.createObjectURL(audioBlob) : '',
       timestamp: new Date().toISOString(),
     }
     appendMessage(newMessage)
-    set({ messageStream: '', isMessageBeingStreamed: false })
+    set({
+      messageStream: '',
+      isMessageBeingStreamed: false,
+      isResponseBeingGenerated: false,
+    })
   },
 }))