diff --git a/.env.example b/.env.example index 39c61de..468e638 100644 --- a/.env.example +++ b/.env.example @@ -4,3 +4,4 @@ EXPO_PUBLIC_TTS_BASE_URL=http://localhost:3333 EXPO_PUBLIC_TTS_VOICE=en-US-JennyNeural EXPO_PUBLIC_TTS_RATE=0.88 EXPO_PUBLIC_TTS_PITCH=+0Hz +EXPO_PUBLIC_STT_BASE_URL=http://localhost:3334 diff --git a/.gitignore b/.gitignore index 42c3acc..a981db0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ dist/ web-build/ expo-env.d.ts .tts-cache/ +.stt-uploads/ +.whisper-models/ # Native .kotlin/ diff --git a/README.md b/README.md index 484705f..ba3a280 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,26 @@ npm run start ## Speech Input And Playback +Speech input can run through local Whisper on the laptop. The iPhone records audio, sends it to the local STT server, Whisper transcribes it to English text, and the app sends that text to Ollama automatically. + +Start the STT server in a third terminal: + +```bash +npm run stt:start +``` + +For Expo Go on iPhone, `.env` must point to the laptop IP: + +```text +EXPO_PUBLIC_STT_BASE_URL=http://192.168.10.33:3334 +``` + +The default local Whisper model is `tiny.en`. It is downloaded on first use and then runs locally without API costs. You can change it with: + +```powershell +$env:STT_MODEL="base.en"; npm run stt:start +``` + Playback uses a local MP3 TTS server on the laptop. AI replies are sent to the laptop, converted to an MP3 with a Microsoft neural English voice, and then played on the iPhone. This avoids the robotic iPhone system voice. Start the TTS server in a second terminal: diff --git a/app.json b/app.json index 86ccb24..2b43c52 100644 --- a/app.json +++ b/app.json @@ -15,7 +15,10 @@ }, "ios": { "supportsTablet": true, - "bundleIdentifier": "com.englishaicoach.app" + "bundleIdentifier": "com.englishaicoach.app", + "infoPlist": { + "NSMicrophoneUsageDescription": "English AI Coach uses the microphone to record your spoken English and transcribe it locally on your laptop." + } }, "android": { "adaptiveIcon": { diff --git a/package-lock.json b/package-lock.json index 66eef66..c3cef48 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,8 @@ "expo-status-bar": "~3.0.9", "express": "^5.2.1", "msedge-tts": "^2.0.5", + "multer": "^2.1.1", + "nodejs-whisper": "^0.3.0", "react": "19.1.0", "react-native": "0.81.5", "react-native-safe-area-context": "~5.6.0", @@ -2712,6 +2714,41 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/@react-native/assets-registry": { "version": "0.81.5", "resolved": "https://registry.npmjs.org/@react-native/assets-registry/-/assets-registry-0.81.5.tgz", @@ -3384,6 +3421,12 @@ "node": ">= 8" } }, + "node_modules/append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==", + "license": "MIT" + }, "node_modules/arg": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", @@ -3920,6 +3963,17 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -4259,6 +4313,21 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "license": "MIT" }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "engines": [ + "node >= 6.0" + ], + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, "node_modules/connect": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/connect/-/connect-3.7.0.tgz", @@ -4674,6 +4743,53 @@ "node": ">=6" } }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/execa/node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/execa/node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/expo": { "version": "54.0.34", "resolved": "https://registry.npmjs.org/expo/-/expo-54.0.34.tgz", @@ -5510,12 +5626,37 @@ "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "license": "MIT" }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", "license": "MIT" }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, "node_modules/fb-watchman": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", @@ -5760,6 +5901,18 @@ "node": ">= 0.4" } }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/getenv": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/getenv/-/getenv-2.0.0.tgz", @@ -5786,6 +5939,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -5927,6 +6092,15 @@ "node": ">= 14" } }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, "node_modules/iconv-lite": { "version": "0.7.2", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", @@ -6073,6 +6247,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -6082,6 +6265,18 @@ "node": ">=8" } }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -6097,6 +6292,18 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -7040,6 +7247,15 @@ "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", "license": "MIT" }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, "node_modules/metro": { "version": "0.83.3", "resolved": "https://registry.npmjs.org/metro/-/metro-0.83.3.tgz", @@ -7573,6 +7789,47 @@ } } }, + "node_modules/multer": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.1.1.tgz", + "integrity": "sha512-mo+QTzKlx8R7E5ylSXxWzGoXoZbOsRMpyitcht8By2KHvMbf3tjwosZ/Mu/XYU6UuJ3VZnODIrak5ZrPiPyB6A==", + "license": "MIT", + "dependencies": { + "append-field": "^1.0.0", + "busboy": "^1.6.0", + "concat-stream": "^2.0.0", + "type-is": "^1.6.18" + }, + "engines": { + "node": ">= 10.16.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/multer/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/multer/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -7638,6 +7895,19 @@ "integrity": "sha512-3qT/88Y3FbH/Kx4szpQQ4HzUbVrHPKTLVpVocKiLfoYvw9XSGOX2FmD2d6DrXbVYyAQTF2HeF6My8jmzx7/CRw==", "license": "MIT" }, + "node_modules/nodejs-whisper": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/nodejs-whisper/-/nodejs-whisper-0.3.0.tgz", + "integrity": "sha512-fIj1Aw6mKevfWKW2wuoFEZuYDRAJPM+L9VWdceGgji3sU8+QKoO8pI30L3+kCYVCCssGOXRzU5qQ0eZdRhfMyg==", + "license": "MIT", + "dependencies": { + "readline-sync": "^1.4.10", + "shelljs": "^0.10.0" + }, + "bin": { + "nodejs-whisper": "dist/downloadModel.js" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7662,6 +7932,18 @@ "node": "^16.14.0 || >=18.0.0" } }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/nullthrows": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/nullthrows/-/nullthrows-1.1.1.tgz", @@ -8183,6 +8465,26 @@ "inherits": "~2.0.3" } }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -8464,6 +8766,15 @@ "node": ">= 6" } }, + "node_modules/readline-sync": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/readline-sync/-/readline-sync-1.4.10.tgz", + "integrity": "sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==", + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/regenerate": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.4.2.tgz", @@ -8621,6 +8932,16 @@ "node": ">=4" } }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -8702,6 +9023,29 @@ "node": ">= 18" } }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -8905,6 +9249,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/shelljs": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.10.0.tgz", + "integrity": "sha512-Jex+xw5Mg2qMZL3qnzXIfaxEtBaC4n7xifqaqtrZDdlheR70OGkydrPJWT0V1cA1k3nanC86x9FwAmQl6w3Klw==", + "license": "BSD-3-Clause", + "dependencies": { + "execa": "^5.1.1", + "fast-glob": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/side-channel": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", @@ -9146,6 +9503,14 @@ "node": ">= 0.10.0" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/strict-uri-encode": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", @@ -9190,6 +9555,15 @@ "node": ">=8" } }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", @@ -9584,6 +9958,12 @@ "url": "https://opencollective.com/express" } }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "license": "MIT" + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", diff --git a/package.json b/package.json index 5878902..348eeaa 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "ios": "expo start --ios", "web": "expo start --web", "prebuild:ios": "expo prebuild --platform ios", - "tts:start": "node tools/tts-server.mjs" + "tts:start": "node tools/tts-server.mjs", + "stt:start": "node tools/stt-server.mjs" }, "dependencies": { "@react-navigation/native": "^7.2.2", @@ -19,6 +20,8 @@ "expo-status-bar": "~3.0.9", "express": "^5.2.1", "msedge-tts": "^2.0.5", + "multer": "^2.1.1", + "nodejs-whisper": "^0.3.0", "react": "19.1.0", "react-native": "0.81.5", "react-native-safe-area-context": "~5.6.0", diff --git a/src/config/stt.ts b/src/config/stt.ts new file mode 100644 index 0000000..5d91869 --- /dev/null +++ b/src/config/stt.ts @@ -0,0 +1,3 @@ +export const sttConfig = { + baseUrl: process.env.EXPO_PUBLIC_STT_BASE_URL ?? '', +}; diff --git a/src/screens/ChatScreen.tsx b/src/screens/ChatScreen.tsx index 73c86be..caabea2 100644 --- a/src/screens/ChatScreen.tsx +++ b/src/screens/ChatScreen.tsx @@ -1,5 +1,6 @@ import type { NativeStackScreenProps } from '@react-navigation/native-stack'; -import { useState } from 'react'; +import { Audio } from 'expo-av'; +import { useRef, useState } from 'react'; import { FlatList, KeyboardAvoidingView, Platform, StyleSheet, Text, TextInput, View } from 'react-native'; import { SafeAreaView } from 'react-native-safe-area-context'; @@ -8,12 +9,14 @@ import { PrimaryButton } from '../components/PrimaryButton'; import type { RootStackParamList } from '../navigation/types'; import { getMockAiReply } from '../services/mockAiCoach'; import { speakText, stopSpeaking } from '../services/speechService'; +import { transcribeAudio } from '../services/sttService'; import { colors } from '../theme/colors'; type ChatScreenProps = NativeStackScreenProps; export function ChatScreen({ route }: ChatScreenProps) { const { level } = route.params; + const recordingRef = useRef(null); const [input, setInput] = useState('I go yesterday to work'); const [messages, setMessages] = useState([ { @@ -23,13 +26,23 @@ export function ChatScreen({ route }: ChatScreenProps) { }, ]); const [isSending, setIsSending] = useState(false); + const [isRecording, setIsRecording] = useState(false); + const [isTranscribing, setIsTranscribing] = useState(false); async function sendMessage() { if (isSending || !input.trim()) { return; } - const userMessage: ChatMessage = { id: `user-${Date.now()}`, role: 'user', text: input.trim() }; + await sendText(input.trim()); + } + + async function sendText(text: string) { + if (isSending || !text.trim()) { + return; + } + + const userMessage: ChatMessage = { id: `user-${Date.now()}`, role: 'user', text: text.trim() }; setMessages((current) => [...current, userMessage]); setInput(''); setIsSending(true); @@ -37,10 +50,73 @@ export function ChatScreen({ route }: ChatScreenProps) { // Uses Ollama when available and falls back to local examples when offline. const reply = await getMockAiReply(userMessage.text, level); setMessages((current) => [...current, { id: `ai-${Date.now()}`, role: 'assistant', text: reply }]); - speakText(reply); + void speakText(reply).catch(() => undefined); setIsSending(false); } + async function toggleRecording() { + if (isRecording) { + await stopRecordingAndSend(); + return; + } + + await startRecording(); + } + + async function startRecording() { + const permission = await Audio.requestPermissionsAsync(); + + if (!permission.granted) { + setInput('Microphone permission is required to record your voice.'); + return; + } + + await Audio.setAudioModeAsync({ + allowsRecordingIOS: true, + playsInSilentModeIOS: true, + }); + + const recording = new Audio.Recording(); + await recording.prepareToRecordAsync(Audio.RecordingOptionsPresets.HIGH_QUALITY); + await recording.startAsync(); + recordingRef.current = recording; + setIsRecording(true); + } + + async function stopRecordingAndSend() { + const recording = recordingRef.current; + + if (!recording) { + return; + } + + setIsRecording(false); + setIsTranscribing(true); + recordingRef.current = null; + + await recording.stopAndUnloadAsync(); + await Audio.setAudioModeAsync({ allowsRecordingIOS: false }); + + const uri = recording.getURI(); + if (!uri) { + setIsTranscribing(false); + return; + } + + try { + const transcript = await transcribeAudio(uri); + setInput(transcript); + + if (transcript) { + await sendText(transcript); + } + } catch { + setInput('I could not transcribe the recording. Please try again.'); + } finally { + setIsTranscribing(false); + } + } + return ( - Tip: tap this field and use the iPhone keyboard microphone to dictate your sentence. + Use Record to send your voice through local Whisper, or use the keyboard microphone for iPhone dictation. + @@ -81,7 +162,7 @@ export function ChatScreen({ route }: ChatScreenProps) { onPress={() => { const lastAiMessage = [...messages].reverse().find((message) => message.role === 'assistant'); if (lastAiMessage) { - speakText(lastAiMessage.text); + void speakText(lastAiMessage.text).catch(() => undefined); } }} /> diff --git a/src/services/sttService.ts b/src/services/sttService.ts new file mode 100644 index 0000000..26eca42 --- /dev/null +++ b/src/services/sttService.ts @@ -0,0 +1,26 @@ +import { sttConfig } from '../config/stt'; + +export async function transcribeAudio(uri: string) { + if (!sttConfig.baseUrl) { + throw new Error('STT server URL is not configured.'); + } + + const formData = new FormData(); + formData.append('audio', { + uri, + name: 'speech.m4a', + type: 'audio/mp4', + } as unknown as Blob); + + const response = await fetch(`${sttConfig.baseUrl}/transcribe`, { + method: 'POST', + body: formData, + }); + + if (!response.ok) { + throw new Error(`STT request failed with ${response.status}`); + } + + const data = (await response.json()) as { text?: string }; + return data.text?.trim() ?? ''; +} diff --git a/tools/stt-server.mjs b/tools/stt-server.mjs new file mode 100644 index 0000000..7f0015b --- /dev/null +++ b/tools/stt-server.mjs @@ -0,0 +1,88 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import cors from 'cors'; +import express from 'express'; +import multer from 'multer'; +import { nodewhisper } from 'nodejs-whisper'; + +const app = express(); +const port = Number(process.env.STT_PORT ?? 3334); +const host = process.env.STT_HOST ?? '0.0.0.0'; +const uploadDir = path.resolve(process.cwd(), '.stt-uploads'); +const modelRootPath = path.resolve(process.cwd(), '.whisper-models'); +const modelName = process.env.STT_MODEL ?? 'tiny.en'; + +fs.mkdirSync(uploadDir, { recursive: true }); +fs.mkdirSync(modelRootPath, { recursive: true }); + +const upload = multer({ dest: uploadDir }); + +app.use(cors()); + +app.get('/health', (_request, response) => { + response.json({ ok: true, model: modelName }); +}); + +app.post('/transcribe', upload.single('audio'), async (request, response) => { + if (!request.file) { + response.status(400).json({ error: 'Missing audio file' }); + return; + } + + try { + const text = await nodewhisper(request.file.path, { + modelName, + autoDownloadModelName: modelName, + modelRootPath, + removeWavFileAfterTranscription: true, + withCuda: false, + logger: quietLogger, + whisperOptions: { + outputInText: false, + outputInSrt: false, + outputInVtt: false, + outputInCsv: false, + outputInJson: false, + outputInJsonFull: false, + outputInLrc: false, + outputInWords: false, + translateToEnglish: false, + splitOnWord: true, + noGpu: true, + }, + }); + + response.json({ text: cleanTranscript(text) }); + } catch (error) { + response.status(500).json({ error: error instanceof Error ? error.message : String(error) }); + } finally { + safeUnlink(request.file.path); + } +}); + +app.listen(port, host, () => { + console.log(`STT server listening on http://${host}:${port}`); + console.log(`Whisper model: ${modelName}`); +}); + +const quietLogger = { + debug: () => undefined, + info: () => undefined, + log: () => undefined, + warn: console.warn, + error: console.error, +}; + +function cleanTranscript(value) { + return String(value) + .replace(/\[[^\]]*\]/g, ' ') + .replace(/\([^)]*\)/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function safeUnlink(filePath) { + if (filePath && fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } +}