From dc4679f072b2f25a4e403e6147256cd928149c8e Mon Sep 17 00:00:00 2001 From: Cfp Date: Wed, 25 Jun 2025 13:21:31 +0200 Subject: [PATCH] feat: TTS with audio ducking --- android/app/capacitor.build.gradle | 3 +- .../de/trafficcue/trafficcue/DuckPlugin.java | 55 +++++++ .../trafficcue/trafficcue/MainActivity.java | 11 +- android/capacitor.settings.gradle | 6 + bun.lock | 43 ++++++ package.json | 3 + src/lib/components/lnv/Sidebar.svelte | 10 ++ src/lib/services/DuckPlugin.ts | 10 ++ src/lib/services/navigation/TTS.ML.ts | 135 ++++++++++++++++++ src/lib/services/navigation/TTS.ts | 46 ++++++ src/lib/services/navigation/TTSWorker.ts | 16 +++ src/lib/services/navigation/routing.svelte.ts | 59 +++++--- 12 files changed, 374 insertions(+), 23 deletions(-) create mode 100644 android/app/src/main/java/de/trafficcue/trafficcue/DuckPlugin.java create mode 100644 src/lib/services/DuckPlugin.ts create mode 100644 src/lib/services/navigation/TTS.ML.ts create mode 100644 src/lib/services/navigation/TTS.ts create mode 100644 src/lib/services/navigation/TTSWorker.ts diff --git a/android/app/capacitor.build.gradle b/android/app/capacitor.build.gradle index bbfb44f..45fe448 100644 --- a/android/app/capacitor.build.gradle +++ b/android/app/capacitor.build.gradle @@ -9,7 +9,8 @@ android { apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle" dependencies { - + implementation project(':capacitor-community-native-audio') + implementation project(':capacitor-community-text-to-speech') } diff --git a/android/app/src/main/java/de/trafficcue/trafficcue/DuckPlugin.java b/android/app/src/main/java/de/trafficcue/trafficcue/DuckPlugin.java new file mode 100644 index 0000000..3222932 --- /dev/null +++ b/android/app/src/main/java/de/trafficcue/trafficcue/DuckPlugin.java @@ -0,0 +1,55 @@ +package de.trafficcue.trafficcue; + +import android.content.Context; +import android.media.AudioFocusRequest; +import android.media.AudioManager; +import android.os.Build; + +import androidx.annotation.RequiresApi; + +import com.getcapacitor.JSObject; +import com.getcapacitor.Plugin; +import com.getcapacitor.PluginCall; +import com.getcapacitor.PluginMethod; +import com.getcapacitor.annotation.CapacitorPlugin; + +@CapacitorPlugin(name = "Duck") +public class DuckPlugin extends Plugin { + + private AudioFocusRequest focusRequest; + private AudioManager audioManager; + + @Override + public void load() { + super.load(); + + audioManager = (AudioManager) getBridge().getActivity().getSystemService(Context.AUDIO_SERVICE); + } + + @RequiresApi(api = Build.VERSION_CODES.P) + @PluginMethod() + public void duck(PluginCall call) { +// String value = call.getString("value"); +// +// JSObject ret = new JSObject(); +// ret.put("value", value); +// call.resolve(ret); + focusRequest = new AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + .setAcceptsDelayedFocusGain(false) + .setWillPauseWhenDucked(false) + .setForceDucking(true) + .build(); + + audioManager.requestAudioFocus(focusRequest); + + call.resolve(); + } + + @RequiresApi(api = Build.VERSION_CODES.O) + @PluginMethod() + public void unduck(PluginCall call) { + audioManager.abandonAudioFocusRequest(focusRequest); + + call.resolve(); + } +} \ No newline at end of file diff --git a/android/app/src/main/java/de/trafficcue/trafficcue/MainActivity.java b/android/app/src/main/java/de/trafficcue/trafficcue/MainActivity.java index 5e2565e..19d6453 100644 --- a/android/app/src/main/java/de/trafficcue/trafficcue/MainActivity.java +++ b/android/app/src/main/java/de/trafficcue/trafficcue/MainActivity.java @@ -1,5 +1,14 @@ package de.trafficcue.trafficcue; +import android.os.Bundle; + import com.getcapacitor.BridgeActivity; -public class MainActivity extends BridgeActivity {} +public class MainActivity extends BridgeActivity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + registerPlugin(DuckPlugin.class); + super.onCreate(savedInstanceState); + } +} diff --git a/android/capacitor.settings.gradle b/android/capacitor.settings.gradle index 9a5fa87..475510e 100644 --- a/android/capacitor.settings.gradle +++ b/android/capacitor.settings.gradle @@ -1,3 +1,9 @@ // DO NOT EDIT THIS FILE! IT IS GENERATED EACH TIME "capacitor update" IS RUN include ':capacitor-android' project(':capacitor-android').projectDir = new File('../node_modules/@capacitor/android/capacitor') + +include ':capacitor-community-native-audio' +project(':capacitor-community-native-audio').projectDir = new File('../node_modules/@capacitor-community/native-audio/android') + +include ':capacitor-community-text-to-speech' +project(':capacitor-community-text-to-speech').projectDir = new File('../node_modules/@capacitor-community/text-to-speech/android') diff --git a/bun.lock b/bun.lock index 6930fa9..52b5a8e 100644 --- a/bun.lock +++ b/bun.lock @@ -4,9 +4,12 @@ "": { "name": "librenav", "dependencies": { + "@capacitor-community/native-audio": "^7.0.0", + "@capacitor-community/text-to-speech": "^6.0.0", "@capacitor/android": "^7.3.0", "@capacitor/cli": "^7.3.0", "@capacitor/core": "^7.3.0", + "@diffusionstudio/vits-web": "^1.0.3", "@eslint/js": "^9.29.0", "eslint": "^9.29.0", "eslint-plugin-svelte": "^3.9.3", @@ -43,6 +46,10 @@ "@babel/runtime": ["@babel/runtime@7.27.3", "", {}, "sha512-7EYtGezsdiDMyY80+65EzwiGmcJqpmcZCojSXaRgdrBaGtWTgDZKq69cPIVped6MkIM78cTQ2GOiEYjwOlG4xw=="], + "@capacitor-community/native-audio": ["@capacitor-community/native-audio@7.0.0", "", { "peerDependencies": { "@capacitor/core": ">=7.0.0" } }, "sha512-wi2l68tU6KDLJWKL6I0lPUKOdB+hNxlvF7RMe/jfkKOMZnd1eUq7kszmbMqyGFZFIxdNG/GMiZrEztZHDypf8A=="], + + "@capacitor-community/text-to-speech": ["@capacitor-community/text-to-speech@6.0.0", "", { "peerDependencies": { "@capacitor/core": ">=7.0.0" } }, "sha512-aKQ+S8q0T3Kqr2PdJI4xLksN8FFOqJI7vlX9sqWrPXPBWa8DQUSSV9M8tvjUq6B9+63er/KSe42ZKMVrD5usKw=="], + "@capacitor/android": ["@capacitor/android@7.3.0", "", { "peerDependencies": { "@capacitor/core": "^7.3.0" } }, "sha512-TqUm+Z3Uk7/rET+adNDKD63JtA0AN+pMQlFRcLPmZJHFmJl6bUOWWELVL0Ixl5XaweTxofNKeDAjjLRIs+wd3g=="], "@capacitor/cli": ["@capacitor/cli@7.3.0", "", { "dependencies": { "@ionic/cli-framework-output": "^2.2.8", "@ionic/utils-subprocess": "^3.0.1", "@ionic/utils-terminal": "^2.3.5", "commander": "^12.1.0", "debug": "^4.4.0", "env-paths": "^2.2.0", "fs-extra": "^11.2.0", "kleur": "^4.1.5", "native-run": "^2.0.1", "open": "^8.4.0", "plist": "^3.1.0", "prompts": "^2.4.2", "rimraf": "^6.0.1", "semver": "^7.6.3", "tar": "^6.1.11", "tslib": "^2.8.1", "xml2js": "^0.6.2" }, "bin": { "cap": "bin/capacitor", "capacitor": "bin/capacitor" } }, "sha512-p0E1ayxw0Njpid8xwOrnuBncdakWxDMbUL2JhDUft38q8tscF2beIIMVhdna1t4Ow55H0r8sdTurwtSjtomrVw=="], @@ -55,6 +62,8 @@ "@deck.gl/mapbox": ["@deck.gl/mapbox@9.1.12", "", { "dependencies": { "@luma.gl/constants": "^9.1.5", "@math.gl/web-mercator": "^4.1.0" }, "peerDependencies": { "@deck.gl/core": "^9.1.0", "@luma.gl/core": "^9.1.5" } }, "sha512-xojtAncKWlURmF4uVqzdqtGkZhHjBWUs6vN1eRUR2ohVhtl5VAqxYGTiUij5gD/2mhoUKw8d53NybLMMN/vbyg=="], + "@diffusionstudio/vits-web": ["@diffusionstudio/vits-web@1.0.3", "", { "dependencies": { "onnxruntime-web": "1.18.0" } }, "sha512-xvOmc3CTWXGKbyiLbsrMczNArm7Z0Bi0h3pij0RTX6SytY/NSqd7h5DdPNUUhBt0nSjWGbo5RIrJLoCQTUQ0nQ=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA=="], "@esbuild/android-arm": ["@esbuild/android-arm@0.25.5", "", { "os": "android", "cpu": "arm" }, "sha512-AdJKSPeEHgi7/ZhuIPtcQKr5RQdo6OO2IL87JkianiMYMPbCtot9fxPbrMiBADOWWm3T2si9stAiVsGbTQFkbA=="], @@ -229,6 +238,26 @@ "@probe.gl/stats": ["@probe.gl/stats@4.1.0", "", {}, "sha512-EI413MkWKBDVNIfLdqbeNSJTs7ToBz/KVGkwi3D+dQrSIkRI2IYbWGAU3xX+D6+CI4ls8ehxMhNpUVMaZggDvQ=="], + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.4", "", {}, "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.41.1", "", { "os": "android", "cpu": "arm" }, "sha512-NELNvyEWZ6R9QMkiytB4/L4zSEaBC03KIXEghptLGLZWJ6VPrL63ooZQCOnlx36aQPGhzuOMwDerC1Eb2VmrLw=="], "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.41.1", "", { "os": "android", "cpu": "arm64" }, "sha512-DXdQe1BJ6TK47ukAoZLehRHhfKnKg9BjnQYUu9gzhI8Mwa1d2fzxA1aw2JixHVl403bwp1+/o/NhhHtxWJBgEA=="], @@ -489,6 +518,8 @@ "flat-cache": ["flat-cache@4.0.1", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.4" } }, "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw=="], + "flatbuffers": ["flatbuffers@1.12.0", "", {}, "sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ=="], + "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], "foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="], @@ -517,6 +548,8 @@ "graphemer": ["graphemer@1.4.0", "", {}, "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="], + "guid-typescript": ["guid-typescript@1.0.9", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="], + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], "i18next": ["i18next@21.10.0", "", { "dependencies": { "@babel/runtime": "^7.17.2" } }, "sha512-YeuIBmFsGjUfO3qBmMOc0rQaun4mIpGKET5WDwvu8lU7gvwpcariZLNtL0Fzj+zazcHUrlXHiptcFhBMFaxzfg=="], @@ -611,6 +644,8 @@ "lodash.merge": ["lodash.merge@4.6.2", "", {}, "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ=="], + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + "lru-cache": ["lru-cache@11.1.0", "", {}, "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A=="], "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="], @@ -647,6 +682,10 @@ "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="], + "onnxruntime-common": ["onnxruntime-common@1.18.0", "", {}, "sha512-lufrSzX6QdKrktAELG5x5VkBpapbCeS3dQwrXbN0eD9rHvU0yAWl7Ztju9FvgAKWvwd/teEKJNj3OwM6eTZh3Q=="], + + "onnxruntime-web": ["onnxruntime-web@1.18.0", "", { "dependencies": { "flatbuffers": "^1.12.0", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.18.0", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-o1UKj4ABIj1gmG7ae0RKJ3/GT+3yoF0RRpfDfeoe0huzRW4FDRLfbkDETmdFAvnJEXuYDE0YT+hhkia0352StQ=="], + "open": ["open@8.4.2", "", { "dependencies": { "define-lazy-prop": "^2.0.0", "is-docker": "^2.1.1", "is-wsl": "^2.2.0" } }, "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ=="], "opening_hours": ["opening_hours@3.8.0", "", { "dependencies": { "i18next": "^21.8.3", "i18next-browser-languagedetector": "^6.1.4", "suncalc": "^1.9.0" } }, "sha512-bRJroECQSe/itVcNmC3j9PPicxn/LBowdd1Hi+4Aa7hCswdt7w81WHfUwrEMbtk1BBYmGJEbSepl8oYYPviSuA=="], @@ -675,6 +714,8 @@ "picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="], + "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "plist": ["plist@3.1.0", "", { "dependencies": { "@xmldom/xmldom": "^0.8.8", "base64-js": "^1.5.1", "xmlbuilder": "^15.1.1" } }, "sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ=="], "pmtiles": ["pmtiles@4.3.0", "", { "dependencies": { "fflate": "^0.8.2" } }, "sha512-wnzQeSiYT/MyO63o7AVxwt7+uKqU0QUy2lHrivM7GvecNy0m1A4voVyGey7bujnEW5Hn+ZzLdvHPoFaqrOzbPA=="], @@ -699,6 +740,8 @@ "prompts": ["prompts@2.4.2", "", { "dependencies": { "kleur": "^3.0.3", "sisteransi": "^1.0.5" } }, "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q=="], + "protobufjs": ["protobufjs@7.5.3", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-sildjKwVqOI2kmFDiXQ6aEB0fjYTafpEvIBs8tOR8qI4spuL9OPROLVu2qZqi/xgCfsHIwVqlaF8JBjWFHnKbw=="], + "protocol-buffers-schema": ["protocol-buffers-schema@3.6.0", "", {}, "sha512-TdDRD+/QNdrCGCE7v8340QyuXd4kIWIgapsE2+n/SaGiSSbomYl4TjHlvIoCWRpE7wFt02EpB35VVA2ImcBVqw=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], diff --git a/package.json b/package.json index 89ba69d..3561245 100644 --- a/package.json +++ b/package.json @@ -31,9 +31,12 @@ }, "type": "module", "dependencies": { + "@capacitor-community/native-audio": "^7.0.0", + "@capacitor-community/text-to-speech": "^6.0.0", "@capacitor/android": "^7.3.0", "@capacitor/cli": "^7.3.0", "@capacitor/core": "^7.3.0", + "@diffusionstudio/vits-web": "^1.0.3", "@eslint/js": "^9.29.0", "eslint": "^9.29.0", "eslint-plugin-svelte": "^3.9.3", diff --git a/src/lib/components/lnv/Sidebar.svelte b/src/lib/components/lnv/Sidebar.svelte index 8956efa..a7a4c5e 100644 --- a/src/lib/components/lnv/Sidebar.svelte +++ b/src/lib/components/lnv/Sidebar.svelte @@ -27,6 +27,7 @@ import * as Popover from "../ui/popover"; import { routing } from "$lib/services/navigation/routing.svelte"; import InRouteSidebar from "./sidebar/InRouteSidebar.svelte"; + import say from "$lib/services/navigation/TTS"; // eslint-disable-next-line @typescript-eslint/no-explicit-any const views: Record> = { @@ -197,6 +198,15 @@ > Join Remote Location + diff --git a/src/lib/services/DuckPlugin.ts b/src/lib/services/DuckPlugin.ts new file mode 100644 index 0000000..8bc7d15 --- /dev/null +++ b/src/lib/services/DuckPlugin.ts @@ -0,0 +1,10 @@ +import { registerPlugin } from "@capacitor/core"; + +export interface DuckPlugin { + duck: () => void; + unduck: () => void; +} + +const Duck = registerPlugin("Duck"); + +export default Duck; \ No newline at end of file diff --git a/src/lib/services/navigation/TTS.ML.ts b/src/lib/services/navigation/TTS.ML.ts new file mode 100644 index 0000000..fd6cdaf --- /dev/null +++ b/src/lib/services/navigation/TTS.ML.ts @@ -0,0 +1,135 @@ +import * as tts from '@diffusionstudio/vits-web'; +import TTSWorker from './TTSWorker.ts?worker'; + +// const VOICE = "en_US-hfc_female-medium"; +const VOICE = "de_DE-thorsten-medium"; + +export async function downloadVoice(): Promise { + await tts.download(VOICE, (progress) => { + console.log(`Downloading ${progress.url} - ${Math.round(progress.loaded * 100 / progress.total)}%`); + }); +} + +interface TTSItem { + text: string; + audio?: Blob; // Optional audio blob if already generated + shouldPlay?: boolean; // Flag to indicate if the audio should be played immediately +} + +const queue: TTSItem[] = []; +let playing = false; +let generating = 0; +const worker = new TTSWorker(); + +worker.addEventListener('message', (event: MessageEvent<{ type: 'result', audio: Blob, text: string }>) => { + if (event.data.type != 'result') return; + + // const audio = new Audio(); + // audio.src = URL.createObjectURL(event.data.audio); + // audio.play(); + // console.log("Audio playing"); + // audio.onended = () => { + // playing = false; + // }; + const item = queue.find(item => item.text === event.data.text); + if (item) { + item.audio = event.data.audio; // Set the audio blob for the item + generating--; + } +}); + +setInterval(() => { + // if(playing) return; + // if(queue[0]) { + // playing = true; + // const text = queue.shift(); + // console.log("Speaking:", text); + // if(text) { + // // tts.predict({ + // // text, + // // voiceId: VOICE, + // // }).then((wav) => { + // // const audio = new Audio(); + // // audio.src = URL.createObjectURL(wav); + // // audio.play(); + // // audio.onended = () => { + // // playing = false; + // // }; + // // }).catch((error) => { + // // console.error("Error playing audio:", error); + // // }); + // worker.postMessage({ + // type: 'init', + // text, + // voiceId: VOICE + // }); + // } + // } + // Pregenerate audio one at a time + if (generating != 0) return; + for (const item of queue) { + // Generate audio blob if it doesn't exist + if (!item.audio) { + generating++; + console.log("Generating audio for:", item.text); + worker.postMessage({ + type: 'init', + text: item.text, + voiceId: VOICE + }); + item.audio = undefined; // Reset audio to undefined until generated + } + } +}, 100); + +setInterval(() => { + if (playing) return; + if (queue.length === 0) return; + + for (const item of queue) { + if (item.shouldPlay && item.audio) { + playing = true; + const audio = new Audio(); + audio.src = URL.createObjectURL(item.audio); + audio.play(); + audio.onended = () => { + playing = false; + }; + queue.splice(queue.indexOf(item), 1); // Remove item from queue after playing + return; // Exit after playing one item + } + } +}, 100); + +export function queueSpeech(text: string) { + // const wav = await tts.predict({ + // text, + // voiceId: VOICE, + // }); + + // const audio = new Audio(); + // audio.src = URL.createObjectURL(wav); + // audio.play(); + if (queue.some(item => item.text === text)) { + // console.warn("Text already in queue, not adding again:", text); + return; + } + console.log("Queuing text for speech:", text); + queue.push({ + text, + shouldPlay: false + }); +} + +export function speak(text: string) { + const existingItem = queue.find(item => item.text === text); + if (existingItem) { + existingItem.shouldPlay = true; + } else { + console.warn("Adding new item to play immediately. Consider queuing instead!"); + queue.push({ + text, + shouldPlay: true + }); + } +} diff --git a/src/lib/services/navigation/TTS.ts b/src/lib/services/navigation/TTS.ts new file mode 100644 index 0000000..a507104 --- /dev/null +++ b/src/lib/services/navigation/TTS.ts @@ -0,0 +1,46 @@ +import type { TextToSpeechPlugin } from "@capacitor-community/text-to-speech"; +import { Capacitor } from "@capacitor/core"; +import Duck from "../DuckPlugin"; + +export let tts: TextToSpeechPlugin | "web" | null = null; + +export async function initTTS() { + if(Capacitor.isNativePlatform()) { + console.log("Using Capacitor TTS"); + tts = (await import("@capacitor-community/text-to-speech")).TextToSpeech; + } else { + console.log("Using Web TTS"); + tts = "web"; + } +} + +export default async function say(text: string) { + if(!tts) { + // alert("TTS not initialized"); + // console.error("TTS not initialized"); + await initTTS(); + // return; + } + console.log("A"); + Duck.duck(); + console.log("B"); + if(tts !== "web") { + try { + await tts?.speak({ + text: text, + lang: "deu-default", // TODO: make this configurable + }); + console.log("C"); + } catch (e) { + console.error("Error speaking text", e); + alert(e); + } + } else { + const utterance = new SpeechSynthesisUtterance(text); + utterance.lang = "de-DE"; + window.speechSynthesis.speak(utterance); + } + console.log("D"); + Duck.unduck(); + console.log("E"); +} diff --git a/src/lib/services/navigation/TTSWorker.ts b/src/lib/services/navigation/TTSWorker.ts new file mode 100644 index 0000000..3c25323 --- /dev/null +++ b/src/lib/services/navigation/TTSWorker.ts @@ -0,0 +1,16 @@ +import * as tts from '@diffusionstudio/vits-web'; + +async function main(event: MessageEvent) { + if (event.data?.type != 'init') return; + + const start = performance.now(); + const blob = await tts.predict({ + text: event.data.text, + voiceId: event.data.voiceId, + }); + console.log('Time taken:', performance.now() - start + ' ms'); + + self.postMessage({ type: 'result', audio: blob, text: event.data.text }); +} + +self.addEventListener('message', main); \ No newline at end of file diff --git a/src/lib/services/navigation/routing.svelte.ts b/src/lib/services/navigation/routing.svelte.ts index 91c6be5..8c56b82 100644 --- a/src/lib/services/navigation/routing.svelte.ts +++ b/src/lib/services/navigation/routing.svelte.ts @@ -1,5 +1,6 @@ import { location } from "$lib/components/lnv/location.svelte"; import { map } from "$lib/components/lnv/map.svelte"; +import say from "./TTS"; import type { ValhallaRequest } from "./ValhallaRequest"; import type { LngLatBoundsLike } from "maplibre-gl"; @@ -156,18 +157,21 @@ async function tickRoute() { } const bgi = currentManeuver.begin_shape_index; - const location = getUserLocation(); + const loc = { + lat: location.lat, + lon: location.lng, + }; const polyline = decodePolyline(trip.legs[0].shape); // Check if the user location is on the last point of the entire route - if (isOnPoint(location, polyline[polyline.length - 1])) { + if (isOnPoint(loc, polyline[polyline.length - 1])) { console.log("Reached destination!"); stopNavigation(); return; } // Check if the user is on the route - if (!isOnShape(location, polyline)) { + if (!isOnShape(loc, polyline)) { console.log("Off route!"); info.isOffRoute = true; // TODO: Implement re-routing logic @@ -180,20 +184,25 @@ async function tickRoute() { currentManeuver.verbal_pre_transition_instruction && !hasAnnouncedPreInstruction ) { - const distanceToEnd = calculateDistance(location, polyline[bgi]); + const distanceToEnd = calculateDistance(loc, polyline[bgi]); // console.log("Distance to end of current maneuver: ", distanceToEnd, " meters"); - if (distanceToEnd <= 100) { + console.log("Speed: ", location.speed, " km/h"); + const verbalDistance = verbalPreInstructionDistance( + location.speed || 50, // Assuming location has a speed property + ); + if (distanceToEnd <= verbalDistance) { hasAnnouncedPreInstruction = true; console.log( "[Verbal instruction] ", currentManeuver.verbal_pre_transition_instruction, ); + say(currentManeuver.verbal_pre_transition_instruction); } } // Check if the user is past the current maneuver // Checks if the user is still on the current maneuver's polyline - if (!isOnShape(location, polyline.slice(bgi))) { + if (!isOnShape(loc, polyline.slice(bgi))) { return; // User is not on the current maneuver's polyline, do not update } @@ -208,7 +217,7 @@ async function tickRoute() { if (currentManeuver.verbal_post_transition_instruction) { hasAnnouncedPreInstruction = false; const distanceToEnd = calculateDistance( - location, + loc, polyline[ trip.legs[0].maneuvers[routing.currentTripInfo.maneuverIdx + 1] .begin_shape_index @@ -219,6 +228,7 @@ async function tickRoute() { "[Verbal instruction] ", currentManeuver.verbal_post_transition_instruction, ); + say(currentManeuver.verbal_post_transition_instruction); } } @@ -232,9 +242,16 @@ async function tickRoute() { info.currentManeuver = trip.legs[0].maneuvers[info.maneuverIdx]; + // queueSpeech(info.currentManeuver.verbal_pre_transition_instruction || ""); + // queueSpeech(info.currentManeuver.verbal_post_transition_instruction || ""); + // TODO: verbal instructions } +function verbalPreInstructionDistance(speed: number): number { + return (speed * 2.222) + 37.144; +} + export function stopNavigation() { if (routing.currentTripInfo.int) { clearInterval(routing.currentTripInfo.int); @@ -245,20 +262,20 @@ export function stopNavigation() { removeAllRoutes(); } -function getUserLocation(): WorldLocation { - // return geolocate.currentLocation!; - return { - lat: location.lat, - lon: location.lng, - }; - // const lnglat = window.geolocate._userLocationDotMarker.getLngLat(); - // return { lat: lnglat.lat, lon: lnglat.lng }; - // console.log(map.value!) - // return { - // lat: 0, - // lon: 0 - // } -} +// function getUserLocation(): WorldLocation { +// // return geolocate.currentLocation!; +// return { +// lat: location.lat, +// lon: location.lng, +// }; +// // const lnglat = window.geolocate._userLocationDotMarker.getLngLat(); +// // return { lat: lnglat.lat, lon: lnglat.lng }; +// // console.log(map.value!) +// // return { +// // lat: 0, +// // lon: 0 +// // } +// } function isOnLine( location: WorldLocation,