Add local Whisper speech input

.env.example

@@ -4,3 +4,4 @@ EXPO_PUBLIC_TTS_BASE_URL=http://localhost:3333
 EXPO_PUBLIC_TTS_VOICE=en-US-JennyNeural
 EXPO_PUBLIC_TTS_RATE=0.88
 EXPO_PUBLIC_TTS_PITCH=+0Hz
+EXPO_PUBLIC_STT_BASE_URL=http://localhost:3334

.gitignore

@@ -9,6 +9,8 @@ dist/
 web-build/
 expo-env.d.ts
 .tts-cache/
+.stt-uploads/
+.whisper-models/
 
 # Native
 .kotlin/

README.md

@@ -109,6 +109,26 @@ npm run start
 
 ## Speech Input And Playback
 
+Speech input can run through local Whisper on the laptop. The iPhone records audio, sends it to the local STT server, Whisper transcribes it to English text, and the app sends that text to Ollama automatically.
+
+Start the STT server in a third terminal:
+
+```bash
+npm run stt:start
+```
+
+For Expo Go on iPhone, `.env` must point to the laptop IP:
+
+```text
+EXPO_PUBLIC_STT_BASE_URL=http://192.168.10.33:3334
+```
+
+The default local Whisper model is `tiny.en`. It is downloaded on first use and then runs locally without API costs. You can change it with:
+
+```powershell
+$env:STT_MODEL="base.en"; npm run stt:start
+```
+
 Playback uses a local MP3 TTS server on the laptop. AI replies are sent to the laptop, converted to an MP3 with a Microsoft neural English voice, and then played on the iPhone. This avoids the robotic iPhone system voice.
 
 Start the TTS server in a second terminal:

app.json

@@ -15,7 +15,10 @@
     },
     "ios": {
       "supportsTablet": true,
-      "bundleIdentifier": "com.englishaicoach.app"
+      "bundleIdentifier": "com.englishaicoach.app",
+      "infoPlist": {
+        "NSMicrophoneUsageDescription": "English AI Coach uses the microphone to record your spoken English and transcribe it locally on your laptop."
+      }
     },
     "android": {
       "adaptiveIcon": {

package-lock.json

@@ -16,6 +16,8 @@
         "expo-status-bar": "~3.0.9",
         "express": "^5.2.1",
         "msedge-tts": "^2.0.5",
+        "multer": "^2.1.1",
+        "nodejs-whisper": "^0.3.0",
         "react": "19.1.0",
         "react-native": "0.81.5",
         "react-native-safe-area-context": "~5.6.0",
@@ -2712,6 +2714,41 @@
         "@jridgewell/sourcemap-codec": "^1.4.14"
       }
     },
+    "node_modules/@nodelib/fs.scandir": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz",
+      "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==",
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.stat": "2.0.5",
+        "run-parallel": "^1.1.9"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.stat": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz",
+      "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
+    "node_modules/@nodelib/fs.walk": {
+      "version": "1.2.8",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz",
+      "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==",
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.scandir": "2.1.5",
+        "fastq": "^1.6.0"
+      },
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/@react-native/assets-registry": {
       "version": "0.81.5",
       "resolved": "https://registry.npmjs.org/@react-native/assets-registry/-/assets-registry-0.81.5.tgz",
@@ -3384,6 +3421,12 @@
         "node": ">= 8"
       }
     },
+    "node_modules/append-field": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
+      "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==",
+      "license": "MIT"
+    },
     "node_modules/arg": {
       "version": "5.0.2",
       "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz",
@@ -3920,6 +3963,17 @@
       "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
       "license": "MIT"
     },
+    "node_modules/busboy": {
+      "version": "1.6.0",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
+      "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==",
+      "dependencies": {
+        "streamsearch": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=10.16.0"
+      }
+    },
     "node_modules/bytes": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz",
@@ -4259,6 +4313,21 @@
       "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
       "license": "MIT"
     },
+    "node_modules/concat-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz",
+      "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==",
+      "engines": [
+        "node >= 6.0"
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "inherits": "^2.0.3",
+        "readable-stream": "^3.0.2",
+        "typedarray": "^0.0.6"
+      }
+    },
     "node_modules/connect": {
       "version": "3.7.0",
       "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz",
@@ -4674,6 +4743,53 @@
         "node": ">=6"
       }
     },
+    "node_modules/execa": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz",
+      "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==",
+      "license": "MIT",
+      "dependencies": {
+        "cross-spawn": "^7.0.3",
+        "get-stream": "^6.0.0",
+        "human-signals": "^2.1.0",
+        "is-stream": "^2.0.0",
+        "merge-stream": "^2.0.0",
+        "npm-run-path": "^4.0.1",
+        "onetime": "^5.1.2",
+        "signal-exit": "^3.0.3",
+        "strip-final-newline": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sindresorhus/execa?sponsor=1"
+      }
+    },
+    "node_modules/execa/node_modules/mimic-fn": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
+      "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/execa/node_modules/onetime": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz",
+      "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==",
+      "license": "MIT",
+      "dependencies": {
+        "mimic-fn": "^2.1.0"
+      },
+      "engines": {
+        "node": ">=6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/expo": {
       "version": "54.0.34",
       "resolved": "https://registry.npmjs.org/expo/-/expo-54.0.34.tgz",
@@ -5510,12 +5626,37 @@
       "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
       "license": "MIT"
     },
+    "node_modules/fast-glob": {
+      "version": "3.3.3",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz",
+      "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==",
+      "license": "MIT",
+      "dependencies": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.2",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.8"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
     "node_modules/fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
       "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
       "license": "MIT"
     },
+    "node_modules/fastq": {
+      "version": "1.20.1",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz",
+      "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==",
+      "license": "ISC",
+      "dependencies": {
+        "reusify": "^1.0.4"
+      }
+    },
     "node_modules/fb-watchman": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz",
@@ -5760,6 +5901,18 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/get-stream": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
+      "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/getenv": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/getenv/-/getenv-2.0.0.tgz",
@@ -5786,6 +5939,18 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/glob-parent": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
+      "license": "ISC",
+      "dependencies": {
+        "is-glob": "^4.0.1"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/gopd": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
@@ -5927,6 +6092,15 @@
         "node": ">= 14"
       }
     },
+    "node_modules/human-signals": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz",
+      "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=10.17.0"
+      }
+    },
     "node_modules/iconv-lite": {
       "version": "0.7.2",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
@@ -6073,6 +6247,15 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-fullwidth-code-point": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
@@ -6082,6 +6265,18 @@
         "node": ">=8"
       }
     },
+    "node_modules/is-glob": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+      "license": "MIT",
+      "dependencies": {
+        "is-extglob": "^2.1.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-number": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
@@ -6097,6 +6292,18 @@
       "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==",
       "license": "MIT"
     },
+    "node_modules/is-stream": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz",
+      "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/is-wsl": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
@@ -7040,6 +7247,15 @@
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
       "license": "MIT"
     },
+    "node_modules/merge2": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 8"
+      }
+    },
     "node_modules/metro": {
       "version": "0.83.3",
       "resolved": "https://registry.npmjs.org/metro/-/metro-0.83.3.tgz",
@@ -7573,6 +7789,47 @@
         }
       }
     },
+    "node_modules/multer": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz",
+      "integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==",
+      "license": "MIT",
+      "dependencies": {
+        "append-field": "^1.0.0",
+        "busboy": "^1.6.0",
+        "concat-stream": "^2.0.0",
+        "type-is": "^1.6.18"
+      },
+      "engines": {
+        "node": ">= 10.16.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/multer/node_modules/media-typer": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+      "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/multer/node_modules/type-is": {
+      "version": "1.6.18",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
+      "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
+      "license": "MIT",
+      "dependencies": {
+        "media-typer": "0.3.0",
+        "mime-types": "~2.1.24"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/mz": {
       "version": "2.7.0",
       "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz",
@@ -7638,6 +7895,19 @@
       "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==",
       "license": "MIT"
     },
+    "node_modules/nodejs-whisper": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/nodejs-whisper/-/nodejs-whisper-0.3.0.tgz",
+      "integrity": "sha512-fIj1Aw6mKevfWKW2wuoFEZuYDRAJPM+L9VWdceGgji3sU8+QKoO8pI30L3+kCYVCCssGOXRzU5qQ0eZdRhfMyg==",
+      "license": "MIT",
+      "dependencies": {
+        "readline-sync": "^1.4.10",
+        "shelljs": "^0.10.0"
+      },
+      "bin": {
+        "nodejs-whisper": "dist/downloadModel.js"
+      }
+    },
     "node_modules/normalize-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -7662,6 +7932,18 @@
         "node": "^16.14.0 || >=18.0.0"
       }
     },
+    "node_modules/npm-run-path": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
+      "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
+      "license": "MIT",
+      "dependencies": {
+        "path-key": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/nullthrows": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz",
@@ -8183,6 +8465,26 @@
         "inherits": "~2.0.3"
       }
     },
+    "node_modules/queue-microtask": {
+      "version": "1.2.3",
+      "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
+      "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT"
+    },
     "node_modules/range-parser": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
@@ -8464,6 +8766,15 @@
         "node": ">= 6"
       }
     },
+    "node_modules/readline-sync": {
+      "version": "1.4.10",
+      "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz",
+      "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.8.0"
+      }
+    },
     "node_modules/regenerate": {
       "version": "1.4.2",
       "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz",
@@ -8621,6 +8932,16 @@
         "node": ">=4"
       }
     },
+    "node_modules/reusify": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz",
+      "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==",
+      "license": "MIT",
+      "engines": {
+        "iojs": ">=1.0.0",
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/rimraf": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
@@ -8702,6 +9023,29 @@
         "node": ">= 18"
       }
     },
+    "node_modules/run-parallel": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
+      "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "queue-microtask": "^1.2.2"
+      }
+    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -8905,6 +9249,19 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/shelljs": {
+      "version": "0.10.0",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.10.0.tgz",
+      "integrity": "sha512-Jex+xw5Mg2qMZL3qnzXIfaxEtBaC4n7xifqaqtrZDdlheR70OGkydrPJWT0V1cA1k3nanC86x9FwAmQl6w3Klw==",
+      "license": "BSD-3-Clause",
+      "dependencies": {
+        "execa": "^5.1.1",
+        "fast-glob": "^3.3.2"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/side-channel": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz",
@@ -9146,6 +9503,14 @@
         "node": ">= 0.10.0"
       }
     },
+    "node_modules/streamsearch": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
+      "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==",
+      "engines": {
+        "node": ">=10.0.0"
+      }
+    },
     "node_modules/strict-uri-encode": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz",
@@ -9190,6 +9555,15 @@
         "node": ">=8"
       }
     },
+    "node_modules/strip-final-newline": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
+      "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/strip-json-comments": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
@@ -9584,6 +9958,12 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/typedarray": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
+      "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==",
+      "license": "MIT"
+    },
     "node_modules/typescript": {
       "version": "5.9.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",

package.json

@@ -8,7 +8,8 @@
     "ios": "expo start --ios",
     "web": "expo start --web",
     "prebuild:ios": "expo prebuild --platform ios",
-    "tts:start": "node tools/tts-server.mjs"
+    "tts:start": "node tools/tts-server.mjs",
+    "stt:start": "node tools/stt-server.mjs"
   },
   "dependencies": {
     "@react-navigation/native": "^7.2.2",
@@ -19,6 +20,8 @@
     "expo-status-bar": "~3.0.9",
     "express": "^5.2.1",
     "msedge-tts": "^2.0.5",
+    "multer": "^2.1.1",
+    "nodejs-whisper": "^0.3.0",
     "react": "19.1.0",
     "react-native": "0.81.5",
     "react-native-safe-area-context": "~5.6.0",

src/config/stt.ts

@@ -0,0 +1,3 @@
+export const sttConfig = {
+  baseUrl: process.env.EXPO_PUBLIC_STT_BASE_URL ?? '',
+};

src/screens/ChatScreen.tsx

@@ -1,5 +1,6 @@
 import type { NativeStackScreenProps } from '@react-navigation/native-stack';
-import { useState } from 'react';
+import { Audio } from 'expo-av';
+import { useRef, useState } from 'react';
 import { FlatList, KeyboardAvoidingView, Platform, StyleSheet, Text, TextInput, View } from 'react-native';
 import { SafeAreaView } from 'react-native-safe-area-context';
 
@@ -8,12 +9,14 @@ import { PrimaryButton } from '../components/PrimaryButton';
 import type { RootStackParamList } from '../navigation/types';
 import { getMockAiReply } from '../services/mockAiCoach';
 import { speakText, stopSpeaking } from '../services/speechService';
+import { transcribeAudio } from '../services/sttService';
 import { colors } from '../theme/colors';
 
 type ChatScreenProps = NativeStackScreenProps<RootStackParamList, 'Chat'>;
 
 export function ChatScreen({ route }: ChatScreenProps) {
   const { level } = route.params;
+  const recordingRef = useRef<Audio.Recording | null>(null);
   const [input, setInput] = useState('I go yesterday to work');
   const [messages, setMessages] = useState<ChatMessage[]>([
     {
@@ -23,13 +26,23 @@ export function ChatScreen({ route }: ChatScreenProps) {
     },
   ]);
   const [isSending, setIsSending] = useState(false);
+  const [isRecording, setIsRecording] = useState(false);
+  const [isTranscribing, setIsTranscribing] = useState(false);
 
   async function sendMessage() {
     if (isSending || !input.trim()) {
       return;
     }
 
-    const userMessage: ChatMessage = { id: `user-${Date.now()}`, role: 'user', text: input.trim() };
+    await sendText(input.trim());
+  }
+
+  async function sendText(text: string) {
+    if (isSending || !text.trim()) {
+      return;
+    }
+
+    const userMessage: ChatMessage = { id: `user-${Date.now()}`, role: 'user', text: text.trim() };
     setMessages((current) => [...current, userMessage]);
     setInput('');
     setIsSending(true);
@@ -37,10 +50,73 @@ export function ChatScreen({ route }: ChatScreenProps) {
     // Uses Ollama when available and falls back to local examples when offline.
     const reply = await getMockAiReply(userMessage.text, level);
     setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]);
-    speakText(reply);
+    void speakText(reply).catch(() => undefined);
     setIsSending(false);
   }
 
+  async function toggleRecording() {
+    if (isRecording) {
+      await stopRecordingAndSend();
+      return;
+    }
+
+    await startRecording();
+  }
+
+  async function startRecording() {
+    const permission = await Audio.requestPermissionsAsync();
+
+    if (!permission.granted) {
+      setInput('Microphone permission is required to record your voice.');
+      return;
+    }
+
+    await Audio.setAudioModeAsync({
+      allowsRecordingIOS: true,
+      playsInSilentModeIOS: true,
+    });
+
+    const recording = new Audio.Recording();
+    await recording.prepareToRecordAsync(Audio.RecordingOptionsPresets.HIGH_QUALITY);
+    await recording.startAsync();
+    recordingRef.current = recording;
+    setIsRecording(true);
+  }
+
+  async function stopRecordingAndSend() {
+    const recording = recordingRef.current;
+
+    if (!recording) {
+      return;
+    }
+
+    setIsRecording(false);
+    setIsTranscribing(true);
+    recordingRef.current = null;
+
+    await recording.stopAndUnloadAsync();
+    await Audio.setAudioModeAsync({ allowsRecordingIOS: false });
+
+    const uri = recording.getURI();
+    if (!uri) {
+      setIsTranscribing(false);
+      return;
+    }
+
+    try {
+      const transcript = await transcribeAudio(uri);
+      setInput(transcript);
+
+      if (transcript) {
+        await sendText(transcript);
+      }
+    } catch {
+      setInput('I could not transcribe the recording. Please try again.');
+    } finally {
+      setIsTranscribing(false);
+    }
+  }
+
   return (
     <SafeAreaView style={styles.safeArea} edges={['bottom']}>
       <KeyboardAvoidingView
@@ -65,8 +141,13 @@ export function ChatScreen({ route }: ChatScreenProps) {
             value={input}
           />
           <Text style={styles.dictationHint}>
-            Tip: tap this field and use the iPhone keyboard microphone to dictate your sentence.
+            Use Record to send your voice through local Whisper, or use the keyboard microphone for iPhone dictation.
           </Text>
+          <PrimaryButton
+            label={isRecording ? 'Stop recording and send' : isTranscribing ? 'Transcribing...' : 'Record voice'}
+            variant={isRecording ? 'primary' : 'secondary'}
+            onPress={toggleRecording}
+          />
           <View style={styles.actionRow}>
             <View style={styles.actionItem}>
               <PrimaryButton label={isSending ? 'Thinking...' : 'Send'} onPress={sendMessage} />
@@ -81,7 +162,7 @@ export function ChatScreen({ route }: ChatScreenProps) {
             onPress={() => {
               const lastAiMessage = [...messages].reverse().find((message) => message.role === 'assistant');
               if (lastAiMessage) {
-                speakText(lastAiMessage.text);
+                void speakText(lastAiMessage.text).catch(() => undefined);
               }
             }}
           />

src/services/sttService.ts

@@ -0,0 +1,26 @@
+import { sttConfig } from '../config/stt';
+
+export async function transcribeAudio(uri: string) {
+  if (!sttConfig.baseUrl) {
+    throw new Error('STT server URL is not configured.');
+  }
+
+  const formData = new FormData();
+  formData.append('audio', {
+    uri,
+    name: 'speech.m4a',
+    type: 'audio/mp4',
+  } as unknown as Blob);
+
+  const response = await fetch(`${sttConfig.baseUrl}/transcribe`, {
+    method: 'POST',
+    body: formData,
+  });
+
+  if (!response.ok) {
+    throw new Error(`STT request failed with ${response.status}`);
+  }
+
+  const data = (await response.json()) as { text?: string };
+  return data.text?.trim() ?? '';
+}

tools/stt-server.mjs

@@ -0,0 +1,88 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import cors from 'cors';
+import express from 'express';
+import multer from 'multer';
+import { nodewhisper } from 'nodejs-whisper';
+
+const app = express();
+const port = Number(process.env.STT_PORT ?? 3334);
+const host = process.env.STT_HOST ?? '0.0.0.0';
+const uploadDir = path.resolve(process.cwd(), '.stt-uploads');
+const modelRootPath = path.resolve(process.cwd(), '.whisper-models');
+const modelName = process.env.STT_MODEL ?? 'tiny.en';
+
+fs.mkdirSync(uploadDir, { recursive: true });
+fs.mkdirSync(modelRootPath, { recursive: true });
+
+const upload = multer({ dest: uploadDir });
+
+app.use(cors());
+
+app.get('/health', (_request, response) => {
+  response.json({ ok: true, model: modelName });
+});
+
+app.post('/transcribe', upload.single('audio'), async (request, response) => {
+  if (!request.file) {
+    response.status(400).json({ error: 'Missing audio file' });
+    return;
+  }
+
+  try {
+    const text = await nodewhisper(request.file.path, {
+      modelName,
+      autoDownloadModelName: modelName,
+      modelRootPath,
+      removeWavFileAfterTranscription: true,
+      withCuda: false,
+      logger: quietLogger,
+      whisperOptions: {
+        outputInText: false,
+        outputInSrt: false,
+        outputInVtt: false,
+        outputInCsv: false,
+        outputInJson: false,
+        outputInJsonFull: false,
+        outputInLrc: false,
+        outputInWords: false,
+        translateToEnglish: false,
+        splitOnWord: true,
+        noGpu: true,
+      },
+    });
+
+    response.json({ text: cleanTranscript(text) });
+  } catch (error) {
+    response.status(500).json({ error: error instanceof Error ? error.message : String(error) });
+  } finally {
+    safeUnlink(request.file.path);
+  }
+});
+
+app.listen(port, host, () => {
+  console.log(`STT server listening on http://${host}:${port}`);
+  console.log(`Whisper model: ${modelName}`);
+});
+
+const quietLogger = {
+  debug: () => undefined,
+  info: () => undefined,
+  log: () => undefined,
+  warn: console.warn,
+  error: console.error,
+};
+
+function cleanTranscript(value) {
+  return String(value)
+    .replace(/\[[^\]]*\]/g, ' ')
+    .replace(/\([^)]*\)/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function safeUnlink(filePath) {
+  if (filePath && fs.existsSync(filePath)) {
+    fs.unlinkSync(filePath);
+  }
+}