add audio scribe sample (#1475)

* add audio scribe sample

* Fix wrong parameter name

* Cleanup and more robust audio scribe

* better manifest descriptions

* demo chat app clean up

.gitignore

@@ -5,4 +5,4 @@ node_modules
 _debug
 _metadata
 dist
-*.swp # vim temp files
+**/*.swp 

functional-samples/ai.gemini-on-device-alt-texter/README.md

@@ -1,4 +1,4 @@
-# On-device multimodal AI with Gemini Nano - image understanding
+# Alt-texter: On-device multimodal AI with Gemini Nano - image understanding
 
 This sample demonstrates how to use the image understanding capabilities of the multi-modal Gemini Nano API preview together with [Chrome's translation API](https://developer.chrome.com/docs/ai/translator-api). To learn more about the API and how to sign-up for the origin trial, head over to [Built-in AI on developer.chrome.com](https://developer.chrome.com/docs/extensions/ai/prompt-api).
 

functional-samples/ai.gemini-on-device-alt-texter/background.js

@@ -35,7 +35,7 @@ chrome.contextMenus.onClicked.addListener(async (info, tab) => {
     ]);
     chrome.runtime.sendMessage({
       action: 'alt-text',
-      text: result.value === 'fulfilled' ? result.value : result.reason.message
+      text: result.status === 'fulfilled' ? result.value : result.reason.message
     });
   }
 });

functional-samples/ai.gemini-on-device-alt-texter/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "Alt Texter",
   "version": "1.0",
-  "description": "Generates alt text for images using the Prompt API.",
+  "description": "Generates alt text for images using the Gemini Nano Prompt API.",
   "permissions": ["contextMenus", "clipboardWrite"],
   "host_permissions": ["<all_urls>"],
   "minimum_chrome_version": "138",

functional-samples/ai.gemini-on-device-audio-scribe/README.md

@@ -0,0 +1,20 @@
+# Audio-Scribe: On-device multimodal AI with Gemini Nano - audio transcription
+
+This sample demonstrates how to use the audio transcription capabilities of the multi-modal Gemini Nano API preview. To learn more about the API and how to sign-up for the origin trial, head over to [Built-in AI on developer.chrome.com](https://developer.chrome.com/docs/extensions/ai/prompt-api).
+
+## Overview
+
+This extension adds a sidepanel that will, when opened, display a transcription of all audio files on a web page (currently it looks only for audio files created using `URL.createObjectUrl`).
+
+## Running this extension
+
+1. Clone this repository.
+1. Load this directory in Chrome as an [unpacked extension](https://developer.chrome.com/docs/extensions/get-started/tutorial/hello-world#load-unpacked).
+1. Open the audio-scribe sidepanel by clicking the audio-scribe action or by pressing the `ALT + A` keyboard shortcut.
+1. Open a chat app in the browser, for example https://web.whatsapp.com/. You can also run the demo chat app via:
+   ```
+   npx serve demo-chat-app
+   ```
+1. All audio messages in the current chat will be transcribed in the side panel.
+
+![Screenshot displaying a demo chat app with a few audio messages. On the right, there is the audio-scribe extension's sidepanel which displayes the transcribed text messages](assets/screenshot.png)

functional-samples/ai.gemini-on-device-audio-scribe/background.js

@@ -0,0 +1,15 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });

functional-samples/ai.gemini-on-device-audio-scribe/bridge.js

@@ -0,0 +1,22 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Forward messages from the content script in the MAIN world to the
+// side panel
+window.addEventListener('message', ({ data }) => {
+  if (data.type !== 'audio-scribe') {
+    return;
+  }
+  chrome.runtime.sendMessage({ data });
+});

functional-samples/ai.gemini-on-device-audio-scribe/demo-chat-app/index.html

@@ -0,0 +1,52 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Chat App Demo</title>
+    <link rel="stylesheet" href="style.css" />
+  </head>
+  <body>
+    <div class="app-container">
+      <div class="sidebar">
+        <!-- Chat list will go here -->
+        <h2>Chats</h2>
+        <ul id="chat-list">
+          <!-- Example chat items -->
+          <li class="chat-item active" data-chat="Alice">
+            <span class="avatar">😊</span>
+            <span>Alice</span>
+          </li>
+          <li class="chat-item" data-chat="Bob">
+            <span class="avatar">😎</span>
+            <span>Bob</span>
+          </li>
+          <li class="chat-item" data-chat="Charlie">
+            <span class="avatar">🥳</span>
+            <span>Charlie</span>
+          </li>
+        </ul>
+      </div>
+      <div class="chat-panel">
+        <div class="chat-header">
+          <!-- Header for the current chat -->
+          <span class="avatar" id="current-chat-avatar">😊</span>
+          <h3 id="current-chat-name">Alice</h3>
+        </div>
+        <div class="message-list" id="message-list">
+          <!-- Messages will be loaded here by JavaScript -->
+        </div>
+        <div class="message-input">
+          <input
+            type="text"
+            id="message-input-field"
+            placeholder="Type a message..."
+          />
+          <button id="send-button">Send</button>
+        </div>
+      </div>
+    </div>
+
+    <script src="script.js"></script>
+  </body>
+</html>

functional-samples/ai.gemini-on-device-audio-scribe/demo-chat-app/script.js

@@ -0,0 +1,224 @@
+document.addEventListener('DOMContentLoaded', () => {
+  const messageList = document.getElementById('message-list');
+
+  const sampleMessages = [
+    { type: 'received', text: '', isAudio: true, audioSrc: 'intro.mp3' },
+    { type: 'sent', text: "I'm in a meeting right now" },
+    { type: 'received', text: '', isAudio: true, audioSrc: 'msg1.mp3' },
+    { type: 'sent', text: '🙄' }
+  ];
+
+  async function renderMessages() {
+    messageList.innerHTML = ''; // Clear existing messages
+    let delay = 0; // Initial delay
+    const delayIncrement = 3000; // 5 seconds in milliseconds
+
+    await timeout(3000);
+
+    sampleMessages.forEach(async (msg, index) => {
+      // Use setTimeout to delay the appearance of each message
+      setTimeout(async () => {
+        const messageElement = document.createElement('div');
+        messageElement.classList.add('message', msg.type);
+
+        // Add a class for fade-in animation (optional, but nice)
+        messageElement.style.opacity = '0'; // Start transparent
+        messageElement.style.transition = 'opacity 0.5s ease-in-out';
+
+        if (msg.isAudio && msg.audioSrc) {
+          messageElement.classList.add('audio');
+
+          // Create audio element (hidden controls)
+          const audioElement = document.createElement('audio');
+          audioElement.preload = 'metadata'; // Important for getting duration
+          const sourceElement = document.createElement('source');
+          sourceElement.type = 'audio/mpeg'; // Assuming MP3
+          audioElement.appendChild(sourceElement);
+
+          const response = await fetch(msg.audioSrc);
+          const data = await response.arrayBuffer();
+          const blob = new Blob([data], { type: 'audio/wav' });
+          sourceElement.src = URL.createObjectURL(blob);
+          // Keep the audio element in the DOM but hidden for playback logic
+          audioElement.style.display = 'none';
+          messageElement.appendChild(audioElement);
+
+          // Create custom controls container
+          const controlsContainer = document.createElement('div');
+          controlsContainer.classList.add('audio-controls');
+
+          // Play/Pause Button
+          const playPauseButton = document.createElement('button');
+          playPauseButton.classList.add('audio-play-pause');
+          playPauseButton.textContent = '▶'; // Play icon initially
+          controlsContainer.appendChild(playPauseButton);
+
+          // Progress Bar (Slider)
+          const progressBar = document.createElement('input');
+          progressBar.type = 'range';
+          progressBar.classList.add('audio-progress');
+          progressBar.value = 0;
+          progressBar.min = 0;
+          progressBar.max = 100; // Will be updated with duration
+          progressBar.step = 0.1;
+          controlsContainer.appendChild(progressBar);
+
+          // Duration Display
+          const durationDisplay = document.createElement('span');
+          durationDisplay.classList.add('audio-duration');
+          durationDisplay.textContent = '0:00'; // Initial display
+          controlsContainer.appendChild(durationDisplay);
+
+          // Append custom controls to the message element
+          messageElement.appendChild(controlsContainer);
+
+          // --- Event Listeners for Custom Controls ---
+
+          // Format time helper function
+          function formatTime(seconds) {
+            const minutes = Math.floor(seconds / 60);
+            const secs = Math.floor(seconds % 60);
+            return `${minutes}:${secs < 10 ? '0' : ''}${secs}`;
+          }
+
+          // Update duration when metadata loads
+          audioElement.addEventListener('loadedmetadata', () => {
+            progressBar.max = audioElement.duration;
+            durationDisplay.textContent = formatTime(audioElement.duration);
+          });
+
+          // Play/Pause functionality
+          playPauseButton.addEventListener('click', () => {
+            if (audioElement.paused) {
+              audioElement.play();
+              playPauseButton.textContent = '❚❚'; // Pause icon
+            } else {
+              audioElement.pause();
+              playPauseButton.textContent = '▶'; // Play icon
+            }
+          });
+
+          // Update progress bar as audio plays
+          audioElement.addEventListener('timeupdate', () => {
+            progressBar.value = audioElement.currentTime;
+            // Update duration display to show current time while playing (optional)
+            durationDisplay.textContent = `${formatTime(audioElement.currentTime)} / ${formatTime(audioElement.duration)}`;
+          });
+
+          // Seek audio when progress bar is changed
+          progressBar.addEventListener('input', () => {
+            audioElement.currentTime = progressBar.value;
+          });
+
+          // Reset button to play when audio ends
+          audioElement.addEventListener('ended', () => {
+            playPauseButton.textContent = '▶';
+            progressBar.value = 0; // Reset progress bar
+          });
+        } else {
+          messageElement.textContent = msg.text;
+          // Check if the message is emoji-only
+          if (isEmojiOnly(msg.text)) {
+            messageElement.classList.add('message-emoji-only');
+          }
+        }
+
+        messageList.appendChild(messageElement);
+
+        // Trigger the fade-in effect
+        requestAnimationFrame(() => {
+          // Ensures the element is in the DOM before changing opacity
+          messageElement.style.opacity = '1';
+        });
+
+        // Scroll to the bottom after adding the message
+        messageList.scrollTop = messageList.scrollHeight;
+      }, index * delayIncrement); // Stagger delay based on index
+    });
+  }
+
+  // Helper function to check if a string contains only emojis
+  function isEmojiOnly(str) {
+    // Regex to match one or more emojis and nothing else
+    const emojiRegex = /^(\p{Emoji_Presentation}|\p{Extended_Pictographic})+$/u;
+    return emojiRegex.test(str.trim());
+  }
+
+  renderMessages();
+
+  // Basic send functionality (optional, just for demo)
+  const sendButton = document.getElementById('send-button'); // Use ID selector
+  const messageInput = document.getElementById('message-input-field'); // Use ID selector
+
+  function sendMessage() {
+    const text = messageInput.value.trim();
+    if (text) {
+      // No need to re-render everything, just add the new message
+      const newMessage = { type: 'sent', text: text };
+      sampleMessages.push(newMessage); // Add to data source
+
+      // Create and append the new message element directly
+      const messageElement = document.createElement('div');
+      messageElement.classList.add('message', 'sent');
+      messageElement.textContent = text;
+
+      // Check if the new message is emoji-only
+      if (isEmojiOnly(text)) {
+        messageElement.classList.add('message-emoji-only');
+      }
+
+      // Add fade-in effect (optional, consistent with renderMessages)
+      messageElement.style.opacity = '0';
+      messageElement.style.transition = 'opacity 0.5s ease-in-out';
+
+      messageList.appendChild(messageElement);
+
+      // Trigger fade-in
+      requestAnimationFrame(() => {
+        messageElement.style.opacity = '1';
+      });
+
+      // Scroll to bottom
+      messageList.scrollTop = messageList.scrollHeight;
+
+      messageInput.value = ''; // Clear input
+    }
+  }
+
+  sendButton.addEventListener('click', sendMessage);
+  messageInput.addEventListener('keypress', (e) => {
+    if (e.key === 'Enter') {
+      sendMessage();
+    }
+  });
+
+  // Chat switching functionality
+  const chatList = document.getElementById('chat-list');
+  const chatItems = chatList.querySelectorAll('.chat-item');
+  const currentChatName = document.getElementById('current-chat-name');
+  const currentChatAvatar = document.getElementById('current-chat-avatar');
+
+  chatItems.forEach((item) => {
+    item.addEventListener('click', () => {
+      // Remove active class from previously active item
+      const currentActive = chatList.querySelector('.chat-item.active');
+      if (currentActive) {
+        currentActive.classList.remove('active');
+      }
+
+      // Add active class to clicked item
+      item.classList.add('active');
+
+      // Update chat header
+      const chatName = item.querySelector('span:not(.avatar)').textContent; // Get name span specifically
+      const avatarEmoji = item.querySelector('.avatar').textContent; // Get emoji from avatar span
+
+      currentChatName.textContent = chatName;
+      currentChatAvatar.textContent = avatarEmoji; // Set emoji in header avatar span
+    });
+  });
+});
+
+function timeout(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

functional-samples/ai.gemini-on-device-audio-scribe/demo-chat-app/style.css

@@ -0,0 +1,280 @@
+/* General Reset and Body Styles */
+body {
+    font-family: 'Segoe UI', 'Helvetica Neue', Helvetica, Arial, sans-serif;
+    margin: 0;
+    background-color: #f0f4f8; /* Lighter, cooler background */
+    height: 100vh;
+    display: flex;
+    overflow: hidden; /* Prevent body scroll */
+}
+
+/* Main App Container */
+.app-container {
+    display: flex;
+    width: 100%;
+    height: 100%;
+    max-width: 1600px;
+    margin: auto;
+    background-color: #ffffff; /* Keep main container white */
+    box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1), 0 2px 8px rgba(0, 0, 0, 0.08); /* Slightly softer shadow */
+}
+
+/* Sidebar (Chat List) */
+.sidebar {
+    width: 30%;
+    min-width: 250px;
+    max-width: 400px;
+    background-color: #ffffff;
+    border-right: 1px solid #e8e8e8; /* Lighter separator */
+    display: flex;
+    flex-direction: column;
+}
+
+.sidebar h2 {
+    padding: 15px 20px;
+    margin: 0;
+    font-size: 18px;
+    font-weight: 500;
+    color: #333; /* Standard dark grey */
+    border-bottom: 1px solid #e8e8e8; /* Lighter separator */
+    background-color: #f8f9fa; /* Very light grey header */
+}
+
+#chat-list {
+    list-style: none;
+    padding: 0;
+    margin: 0;
+    overflow-y: auto; /* Allow scrolling for chat list */
+    flex-grow: 1;
+}
+
+/* Style for avatar emojis */
+.avatar {
+    width: 40px;
+    height: 40px;
+    border-radius: 50%; /* Make avatars circular */
+    margin-right: 10px; /* Space between avatar and name */
+    display: flex; /* Use flexbox for centering */
+    align-items: center;
+    justify-content: center;
+    font-size: 24px; /* Adjust emoji size */
+    background-color: #eee; /* Optional: Add a background */
+    flex-shrink: 0; /* Prevent shrinking */
+}
+
+.chat-item {
+    padding: 10px 15px; /* Slightly less padding */
+    border-bottom: 1px solid #f0f0f0; /* Lighter separator */
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    /* gap: 15px; */ /* Replaced by avatar margin */
+}
+
+.chat-item:hover {
+    background-color: #f0f4f8; /* Light blue hover */
+}
+
+.chat-item.active {
+    background-color: #e2eaf1; /* Slightly darker blue for active */
+}
+
+/* Chat Panel (Conversation View) */
+.chat-panel {
+    flex-grow: 1;
+    display: flex;
+    flex-direction: column;
+    background-color: #e8f0f4; /* Lighter blue chat background */
+}
+
+.chat-header {
+    padding: 10px 16px;
+    background-color: #f8f9fa; /* Match sidebar header */
+    border-bottom: 1px solid #e8e8e8; /* Lighter separator */
+    display: flex;
+    align-items: center;
+    min-height: 40px;
+}
+
+/* No specific override needed for header avatar anymore if base style is good */
+/* .chat-header .avatar { ... } */ /* Removed redundant/conflicting styles */
+
+.chat-header h3 {
+    margin: 0;
+    font-size: 16px;
+    font-weight: 500;
+    color: #333; /* Standard dark grey */
+}
+
+/* Message List */
+.message-list {
+    flex-grow: 1;
+    padding: 20px 5%; /* Padding relative to width */
+    overflow-y: auto;
+    display: flex;
+    flex-direction: column;
+    gap: 5px; /* Smaller gap between messages */
+}
+
+/* Individual Messages */
+.message {
+    padding: 6px 12px;
+    border-radius: 7.5px; /* WhatsApp's bubble radius */
+    max-width: 65%;
+    word-wrap: break-word;
+    box-shadow: 0 1px 1px rgba(0, 0, 0, 0.08); /* Softer shadow */
+    font-size: 14.5px;
+    line-height: 1.4;
+}
+
+.message.sent {
+    background-color: #cce5ff; /* Light blue for sent messages */
+    color: #004085; /* Darker blue text for contrast */
+    align-self: flex-end;
+    margin-left: auto;
+}
+
+.message.received {
+    background-color: #ffffff; /* White for received */
+    color: #333; /* Standard dark text */
+    align-self: flex-start;
+    margin-right: auto;
+}
+
+/* Style for messages containing only emojis */
+.message.message-emoji-only {
+    font-size: 29px; /* Double the base message font size */
+    line-height: 1.2; /* Adjust line height for larger font */
+    padding: 2px 6px; /* Adjust padding slightly */
+    /* Optional: Remove background for pure emoji look */
+    /* background-color: transparent; */
+    /* box-shadow: none; */
+}
+
+/* Message Input Area */
+.message-input {
+    display: flex;
+    align-items: center;
+    padding: 10px 20px;
+    background-color: #f8f9fa; /* Match header background */
+    border-top: 1px solid #e8e8e8; /* Lighter separator */
+}
+
+#message-input-field { /* Use the ID from HTML */
+    flex-grow: 1;
+    padding: 10px 15px;
+    border: none; /* Remove default border */
+    border-radius: 20px; /* Rounded input */
+    margin-right: 10px;
+    font-size: 15px;
+    outline: none;
+}
+
+#send-button {
+    padding: 10px 15px;
+    background-color: #007bff; /* Standard blue send button */
+    color: white;
+    border: none;
+    border-radius: 20px; /* Rounded corners like input */
+    cursor: pointer;
+    font-size: 15px; /* Match input field font size */
+    /* Remove fixed width and height */
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    /* Consider adding an SVG icon here */
+}
+
+#send-button:hover {
+    background-color: #0056b3; /* Darker blue on hover */
+}
+
+/* Custom Audio Player Styles */
+.message.audio {
+    /* Adjust padding if needed for controls */
+    padding-top: 8px;
+    padding-bottom: 8px;
+}
+
+.audio-controls {
+    display: flex;
+    align-items: center;
+    gap: 10px; /* Space between button, progress, duration */
+    width: 100%; /* Ensure controls take available width */
+    margin-top: 5px; /* Space above controls if there was text */
+}
+
+.audio-play-pause {
+    background-color: #007bff; /* Match send button color */
+    color: white;
+    border: none;
+    border-radius: 50%; /* Circular button */
+    width: 30px;
+    height: 30px;
+    font-size: 14px; /* Adjust icon size */
+    cursor: pointer;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    padding: 0; /* Remove default padding */
+    flex-shrink: 0; /* Prevent button from shrinking */
+}
+
+.audio-play-pause:hover {
+    background-color: #0056b3; /* Darker blue on hover */
+}
+
+.audio-progress {
+    flex-grow: 1; /* Take up remaining space */
+    height: 6px; /* Slimmer progress bar */
+    cursor: pointer;
+    appearance: none; /* Override default look */
+    background: #ddd; /* Track background */
+    border-radius: 3px;
+    outline: none;
+}
+
+/* Styling the progress bar thumb (the draggable part) */
+.audio-progress::-webkit-slider-thumb {
+    appearance: none;
+    width: 12px;
+    height: 12px;
+    background: #007bff; /* Thumb color */
+    border-radius: 50%;
+    cursor: pointer;
+    margin-top: -3px; /* Adjust vertical alignment ( (track_height - thumb_height) / 2 ) */
+}
+
+.audio-progress::-moz-range-thumb {
+    width: 12px;
+    height: 12px;
+    background: #007bff;
+    border-radius: 50%;
+    cursor: pointer;
+    border: none; /* Remove default border in Firefox */
+}
+
+/* Styling the progress bar track */
+.audio-progress::-webkit-slider-runnable-track {
+    height: 6px;
+    background: #ccc; /* Slightly darker track */
+    border-radius: 3px;
+}
+
+.audio-progress::-moz-range-track {
+    height: 6px;
+    background: #ccc;
+    border-radius: 3px;
+}
+
+.audio-duration {
+    font-size: 12px;
+    color: #555; /* Muted color for duration */
+    min-width: 35px; /* Ensure space for MM:SS */
+    text-align: right;
+    flex-shrink: 0; /* Prevent duration from shrinking */
+}
+
+#chat {
+    background-color: red;
+}

functional-samples/ai.gemini-on-device-audio-scribe/manifest.json

@@ -0,0 +1,40 @@
+{
+  "name": "Audio scribe",
+  "description": "Use Gemini Nano to transcribe audio messages in chat conversations.",
+  "version": "0.2",
+  "icons": {
+    "128": "assets/icon128.png"
+  },
+  "background": {
+    "service_worker": "background.js"
+  },
+  "content_scripts": [
+    {
+      "matches": ["<all_urls>"],
+      "js": ["override-createobject-url.js"],
+      "run_at": "document_start",
+      "world": "MAIN"
+    },
+    {
+      "matches": ["<all_urls>"],
+      "js": ["bridge.js"],
+      "run_at": "document_start"
+    }
+  ],
+  "action": {},
+  "side_panel": {
+    "default_path": "sidepanel.html"
+  },
+  "permissions": ["sidePanel"],
+  "commands": {
+    "_execute_action": {
+      "suggested_key": {
+        "windows": "Alt+A",
+        "mac": "Alt+A",
+        "chromeos": "Alt+A",
+        "linux": "Alt+A"
+      }
+    }
+  },
+  "manifest_version": 3
+}

functional-samples/ai.gemini-on-device-audio-scribe/override-createobject-url.js

@@ -0,0 +1,24 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const originalCreateObjectURL = URL.createObjectURL;
+
+// Signal the creation of object URLs to the side panel
+// Note: you should be only doing this for specific websites
+// and not for all (as we do in this demo)
+URL.createObjectURL = (object) => {
+  const objectUrl = originalCreateObjectURL.call(URL, object);
+  window.postMessage({ type: 'audio-scribe', objectUrl });
+  return objectUrl;
+};

functional-samples/ai.gemini-on-device-audio-scribe/sidepanel.html

@@ -0,0 +1,47 @@
+<!doctype html>
+<html>
+  <head>
+    <style>
+      @import 'https://unpkg.com/open-props';
+      @import 'https://unpkg.com/open-props/normalize.min.css';
+      @import 'https://unpkg.com/open-props/buttons.min.css';
+      @import 'https://unpkg.com/open-props/theme.light.switch.min.css';
+      @import 'https://unpkg.com/open-props/theme.dark.switch.min.css';
+
+      :root {
+        --font-size-00: 0.6rem;
+      }
+      body {
+        margin: auto;
+        padding: var(--size-2);
+      }
+      ul {
+        padding: var(--size-2);
+      }
+      li {
+        background: var(--surface-3);
+        border: 1px solid var(--surface-1);
+        padding: var(--size-4);
+        margin-bottom: var(--size-3);
+        border-radius: var(--radius-3);
+        box-shadow: var(--shadow-2);
+        list-style: none;
+        border-radius: var(--radius-2);
+        padding: var(--size-fluid-3);
+        box-shadow: var(--shadow-2);
+
+        &:hover {
+          box-shadow: var(--shadow-3);
+        }
+
+        @media (--motionOK) {
+          animation: var(--animation-fade-in);
+        }
+      }
+    </style>
+  </head>
+  <body>
+    <ul id="messages"></ul>
+    <script src="sidepanel.js"></script>
+  </body>
+</html>

functional-samples/ai.gemini-on-device-audio-scribe/sidepanel.js

@@ -0,0 +1,78 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global LanguageModel */
+
+chrome.runtime.onMessage.addListener(async ({ data }) => {
+  let content;
+  try {
+    if (data.type != 'audio-scribe' || !data || !isValidUrl(data.objectUrl)) {
+      return;
+    }
+    // Check if it's an audio file
+    const audio = await fetch(data.objectUrl);
+    content = await audio.blob();
+    if (!content.type || !content.type.startsWidth('audio/')) {
+      return;
+    }
+  } catch (e) {
+    console.log(e);
+  }
+
+  // Setup message UI
+  const messages = document.getElementById('messages');
+  const li = document.createElement('li');
+  li.append('...');
+  messages.append(li);
+
+  try {
+    // Transcribe audio
+    const availability = await LanguageModel.availability();
+    if (availability !== 'available') {
+      console.error('Model is', availability);
+      throw new Error('Model is not available');
+    }
+    const session = await LanguageModel.create({
+      expectedInputs: [{ type: 'audio' }]
+    });
+    const stream = session.promptStreaming([
+      { type: 'audio', content },
+      'transcribe this audio'
+    ]);
+
+    // Render streamed response
+    let first = true;
+    for await (const chunk of stream) {
+      if (first) {
+        li.textContent = '';
+        first = false;
+      }
+      li.append(chunk);
+    }
+  } catch (error) {
+    console.log(error);
+    li.textContent = error.message;
+  }
+});
+
+function isValidUrl(string) {
+  let url;
+
+  try {
+    url = new URL(string);
+    return true;
+  } catch (_) {
+    return false;
+  }
+}

functional-samples/ai.gemini-on-device-calendar-mate/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "Calendar Mate",
   "version": "1.0",
-  "description": "Create Google Calendar events from selected text.",
+  "description": "Create Google Calendar events from selected text using Gemini Nano.",
   "permissions": ["contextMenus"],
   "minimum_chrome_version": "138",
   "background": {

functional-samples/ai.gemini-on-device/manifest.json

@@ -2,7 +2,7 @@
   "name": "Chrome Prompt AI Demo",
   "version": "0.2",
   "manifest_version": 3,
-  "description": "Try Chrome's built-in prompt API.",
+  "description": "Try Chrome's built-in prompt API built with Gemini Nano.",
   "background": {
     "service_worker": "background.js"
   },