From 3da16940859f589603aae267bdc54279968119d2 Mon Sep 17 00:00:00 2001 From: Luke The Dev <252071647+iamlukethedev@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:35:19 -0500 Subject: [PATCH] feat: add SOUNDCLAW jukebox skill integration (#67) Add the office jukebox flow so Spotify can be controlled from the SOUNDCLAW skill, manual jukebox UI, and local browser auth bridge during development. Made-with: Cursor --- .gitignore | 3 + README.md | 21 + assets/skills/soundclaw/SKILL.md | 92 + package-lock.json | 277 +++ package.json | 2 + server/index.js | 67 +- src/app/spotify/callback/page.tsx | 39 + src/features/office/screens/OfficeScreen.tsx | 220 +++ src/features/retro-office/RetroOffice3D.tsx | 1739 +++++++++-------- .../retro-office/core/furnitureDefaults.ts | 43 +- src/features/retro-office/core/geometry.ts | 2 + src/features/retro-office/core/navigation.ts | 11 +- src/features/retro-office/core/types.ts | 2 +- src/features/retro-office/objects/Jukebox.tsx | 228 +++ src/features/retro-office/objects/agents.tsx | 153 +- src/features/spotify-jukebox/agentBridge.ts | 101 + src/features/spotify-jukebox/auth.ts | 188 ++ .../components/JukeboxDisabledPanel.tsx | 45 + .../components/JukeboxPanel.tsx | 463 +++++ src/features/spotify-jukebox/spotifyApi.ts | 115 ++ src/features/spotify-jukebox/store.ts | 182 ++ src/lib/office/eventTriggers.ts | 236 ++- src/lib/office/places.ts | 27 +- src/lib/skills/catalog.ts | 40 +- src/lib/skills/marketplace.ts | 88 +- src/lib/skills/packaged.ts | 57 +- tests/unit/skillTriggers.test.ts | 13 +- 27 files changed, 3471 insertions(+), 983 deletions(-) create mode 100644 assets/skills/soundclaw/SKILL.md create mode 100644 src/app/spotify/callback/page.tsx create mode 100644 src/features/retro-office/objects/Jukebox.tsx create mode 100644 src/features/spotify-jukebox/agentBridge.ts create mode 100644 src/features/spotify-jukebox/auth.ts create mode 100644 src/features/spotify-jukebox/components/JukeboxDisabledPanel.tsx create mode 100644 src/features/spotify-jukebox/components/JukeboxPanel.tsx create mode 100644 src/features/spotify-jukebox/spotifyApi.ts create mode 100644 src/features/spotify-jukebox/store.ts diff --git a/.gitignore b/.gitignore index 933631a..35ff3e9 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,6 @@ test-results # bv (beads viewer) local config and caches .bv/ + +# Local HTTPS development certificates (generated by dev:https). +.certs/ diff --git a/README.md b/README.md index 75b64d7..e91a14c 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,7 @@ See [`.env.example`](.env.example) for the full local development template. - The immersive retro office (`/office`) and the Phaser builder (`/office/builder`) are related but still separate stacks. - The app keeps gateway secrets out of browser persistent storage, but the current connection flow still loads the upstream URL/token into browser memory at runtime. +- Local Spotify auth for `SOUNDCLAW` currently stores an access token only. Refresh-token handling is not implemented yet, so local Spotify auth may need to be repeated after the token expires. ## Troubleshooting @@ -205,6 +206,26 @@ If the UI loads but Connect fails, the problem is usually on the Studio -> Gatew Marketplace skill installs now use a gateway-native workspace flow and do not require enabling SSH on the user machine. +### Spotify auth on localhost + +If you are testing the `SOUNDCLAW` jukebox locally and Spotify OAuth does not accept your `localhost` callback, use an `ngrok` callback bridge: + +1. Keep Claw3D running locally on `http://localhost:3000`. +2. Start `ngrok` for the local Studio server, for example `ngrok http 3000`. +3. In the jukebox setup UI, paste your public `ngrok` URL into the `ngrok Public URL` field. +4. In the Spotify developer dashboard, register `https:///spotify/callback` as the redirect URI. +5. Complete Spotify sign-in from the jukebox panel. + +How it works: + +- The main Claw3D app stays on `localhost`, so your normal local office state and agent state remain intact. +- Spotify redirects to the `ngrok` callback URL. +- The callback page passes the auth code back to the open local Claw3D window. + +Current local limitation: + +- Because only the Spotify access token is stored right now, you may need to repeat the `ngrok` auth flow when that token expires during local development. + If you use other advanced gateway-host operations over SSH: - macOS: enable `System Settings` -> `General` -> `Sharing` -> `Remote Login`, and make sure the target user is allowed. diff --git a/assets/skills/soundclaw/SKILL.md b/assets/skills/soundclaw/SKILL.md new file mode 100644 index 0000000..a3a455b --- /dev/null +++ b/assets/skills/soundclaw/SKILL.md @@ -0,0 +1,92 @@ +--- +name: soundclaw +description: Control Spotify playback, search music, and return shareable music links. +metadata: {"openclaw":{"skillKey":"soundclaw"}} +--- + +# SOUNDCLAW + +Use this skill when the user wants an agent to search for music, play a song or playlist, control Spotify playback, or send back a shareable Spotify link on the same channel the request came from. + +## Trigger + +```json +{ + "activation": { + "anyPhrases": [ + "spotify", + "play a song", + "play this song", + "play music", + "play a playlist", + "find a song", + "queue this song", + "music link" + ] + }, + "movement": { + "target": "jukebox", + "skipIfAlreadyThere": true + } +} +``` + +When this skill is activated, the agent should walk to the office jukebox before handling the request. + +- Treat requests from Telegram or any other external surface as valid triggers when they ask for Spotify playback, search, queueing, or music-link sharing. +- The physical behavior for this skill is: go to the jukebox, perform the music-selection workflow, then report the result. +- If the agent is already at the jukebox, continue without adding extra movement narration. + +## Channel behavior + +- Reply on the same active channel or session that received the request. +- If playback cannot start but a matching track, album, or playlist is found, send back the best Spotify link instead of failing silently. +- If multiple matches are plausible, ask a clarifying question instead of guessing. + +--- + +## OpenClaw Gateway Skill Contract + +> This section is for developers implementing the backend skill handler in OpenClaw. +> The Claw3D UI handles authentication via Spotify PKCE OAuth in the browser. +> The gateway skill handles agent-driven requests via the `soundclaw.*` RPC namespace. + +### Authentication model + +The user authenticates directly in the browser (PKCE, no secret required). +The access token is stored in browser `localStorage` under the key `soundclaw_token`. + +For **agent-driven** playback (e.g. "play Jazz for me"), the gateway skill should either: +- Use a server-side Spotify app token (Client Credentials) for search-only actions, or +- Instruct the agent to tell the user to use the jukebox panel for actual playback + +### RPC methods the gateway skill should expose + +```ts +// Search for tracks. Returns a list of { name, artist, album, uri, spotifyUrl }. +soundclaw.search({ query: string }): SpotifySearchResult[] + +// Get a shareable Spotify link for a query (for Telegram/chat replies). +soundclaw.getLink({ query: string }): { url: string; title: string } + +// Report current playback state (reads from Spotify API). +soundclaw.playerStatus(): PlayerStatus | null + +// Request playback of a URI (requires user to be authenticated in browser). +soundclaw.play({ uri: string }): { ok: boolean; message?: string } + +// Pause / resume / skip. +soundclaw.pause(): void +soundclaw.resume(): void +soundclaw.next(): void +soundclaw.previous(): void +``` + +### Agent workflow + +1. Agent receives a music request ("play some jazz", "find this song", etc.) +2. Agent walks to the jukebox (`movement.target: "jukebox"`) +3. Agent calls `soundclaw.search` to find the best match +4. If the request came from a chat channel (Telegram, etc.): call `soundclaw.getLink` and reply with the link +5. If the request came from the office UI: call `soundclaw.play` to start playback +6. Agent reports back what was played or linked diff --git a/package-lock.json b/package-lock.json index 5cc8ff7..a8a650d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,7 @@ "eslint-config-next": "16.1.6", "eslint-config-prettier": "^10.1.8", "jsdom": "^27.4.0", + "selfsigned": "^5.5.0", "tailwindcss": "^4", "tw-animate-css": "^1.4.0", "typescript": "^5", @@ -2090,6 +2091,165 @@ "node": ">=14" } }, + "node_modules/@peculiar/asn1-cms": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-cms/-/asn1-cms-2.6.1.tgz", + "integrity": "sha512-vdG4fBF6Lkirkcl53q6eOdn3XYKt+kJTG59edgRZORlg/3atWWEReRCx5rYE1ZzTTX6vLK5zDMjHh7vbrcXGtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "@peculiar/asn1-x509-attr": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-csr": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-csr/-/asn1-csr-2.6.1.tgz", + "integrity": "sha512-WRWnKfIocHyzFYQTka8O/tXCiBquAPSrRjXbOkHbO4qdmS6loffCEGs+rby6WxxGdJCuunnhS2duHURhjyio6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-ecc": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-ecc/-/asn1-ecc-2.6.1.tgz", + "integrity": "sha512-+Vqw8WFxrtDIN5ehUdvlN2m73exS2JVG0UAyfVB31gIfor3zWEAQPD+K9ydCxaj3MLen9k0JhKpu9LqviuCE1g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pfx": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pfx/-/asn1-pfx-2.6.1.tgz", + "integrity": "sha512-nB5jVQy3MAAWvq0KY0R2JUZG8bO/bTLpnwyOzXyEh/e54ynGTatAR+csOnXkkVD9AFZ2uL8Z7EV918+qB1qDvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.1", + "@peculiar/asn1-pkcs8": "^2.6.1", + "@peculiar/asn1-rsa": "^2.6.1", + "@peculiar/asn1-schema": "^2.6.0", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pkcs8": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs8/-/asn1-pkcs8-2.6.1.tgz", + "integrity": "sha512-JB5iQ9Izn5yGMw3ZG4Nw3Xn/hb/G38GYF3lf7WmJb8JZUydhVGEjK/ZlFSWhnlB7K/4oqEs8HnfFIKklhR58Tw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-pkcs9": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-pkcs9/-/asn1-pkcs9-2.6.1.tgz", + "integrity": "sha512-5EV8nZoMSxeWmcxWmmcolg22ojZRgJg+Y9MX2fnE2bGRo5KQLqV5IL9kdSQDZxlHz95tHvIq9F//bvL1OeNILw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.1", + "@peculiar/asn1-pfx": "^2.6.1", + "@peculiar/asn1-pkcs8": "^2.6.1", + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "@peculiar/asn1-x509-attr": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-rsa": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-rsa/-/asn1-rsa-2.6.1.tgz", + "integrity": "sha512-1nVMEh46SElUt5CB3RUTV4EG/z7iYc7EoaDY5ECwganibQPkZ/Y2eMsTKB/LeyrUJ+W/tKoD9WUqIy8vB+CEdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-schema": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-schema/-/asn1-schema-2.6.0.tgz", + "integrity": "sha512-xNLYLBFTBKkCzEZIw842BxytQQATQv+lDTCEMZ8C196iJcJJMBUZxrhSTxLaohMyKK8QlzRNTRkUmanucnDSqg==", + "dev": true, + "license": "MIT", + "dependencies": { + "asn1js": "^3.0.6", + "pvtsutils": "^1.3.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509/-/asn1-x509-2.6.1.tgz", + "integrity": "sha512-O9jT5F1A2+t3r7C4VT7LYGXqkGLK7Kj1xFpz7U0isPrubwU5PbDoyYtx6MiGst29yq7pXN5vZbQFKRCP+lLZlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "asn1js": "^3.0.6", + "pvtsutils": "^1.3.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/asn1-x509-attr": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/@peculiar/asn1-x509-attr/-/asn1-x509-attr-2.6.1.tgz", + "integrity": "sha512-tlW6cxoHwgcQghnJwv3YS+9OO1737zgPogZ+CgWRUK4roEwIPzRH4JEiG770xe5HX2ATfCpmX60gurfWIF9dcQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.1", + "asn1js": "^3.0.6", + "tslib": "^2.8.1" + } + }, + "node_modules/@peculiar/x509": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@peculiar/x509/-/x509-1.14.3.tgz", + "integrity": "sha512-C2Xj8FZ0uHWeCXXqX5B4/gVFQmtSkiuOolzAgutjTfseNOHT3pUjljDZsTSxXFGgio54bCzVFqmEOUrIVk8RDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/asn1-cms": "^2.6.0", + "@peculiar/asn1-csr": "^2.6.0", + "@peculiar/asn1-ecc": "^2.6.0", + "@peculiar/asn1-pkcs9": "^2.6.0", + "@peculiar/asn1-rsa": "^2.6.0", + "@peculiar/asn1-schema": "^2.6.0", + "@peculiar/asn1-x509": "^2.6.0", + "pvtsutils": "^1.3.6", + "reflect-metadata": "^0.2.2", + "tslib": "^2.8.1", + "tsyringe": "^4.10.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@playwright/test": { "version": "1.58.0", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.0.tgz", @@ -4083,6 +4243,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/asn1js": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/asn1js/-/asn1js-3.0.7.tgz", + "integrity": "sha512-uLvq6KJu04qoQM6gvBfKFjlh6Gl0vOKQuR5cJMDHQkmwfMOQeN3F3SHCv9SNYSL+CRoHvOGFfllDlVz03GQjvQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.3", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -4283,6 +4458,16 @@ "ieee754": "^1.2.1" } }, + "node_modules/bytestreamjs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/bytestreamjs/-/bytestreamjs-2.0.1.tgz", + "integrity": "sha512-U1Z/ob71V/bXfVABvNr/Kumf5VyeQRBEm6Txb0PQ6S7V5GpBM3w4Cbqz/xPDicR5tN0uvDifng8C+5qECeGwyQ==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -8783,6 +8968,37 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/pkijs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pkijs/-/pkijs-3.4.0.tgz", + "integrity": "sha512-emEcLuomt2j03vxD54giVB4SxTjnsqkU692xZOZXHDVoYyypEm+b3jpiTcc+Cf+myooc+/Ly0z01jqeNHVgJGw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@noble/hashes": "1.4.0", + "asn1js": "^3.0.6", + "bytestreamjs": "^2.0.1", + "pvtsutils": "^1.3.6", + "pvutils": "^1.1.3", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/pkijs/node_modules/@noble/hashes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.4.0.tgz", + "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/playwright": { "version": "1.58.0", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.0.tgz", @@ -8950,6 +9166,26 @@ "node": ">=6" } }, + "node_modules/pvtsutils": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/pvtsutils/-/pvtsutils-1.3.6.tgz", + "integrity": "sha512-PLgQXQ6H2FWCaeRak8vvk1GW462lMxB5s3Jm673N82zI4vqtVUPuZdffdZbPDFRoU8kAhItWFtPCWiPpp4/EDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.8.1" + } + }, + "node_modules/pvutils": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/pvutils/-/pvutils-1.1.5.tgz", + "integrity": "sha512-KTqnxsgGiQ6ZAzZCVlJH5eOjSnvlyEgx1m8bkRJfOhmGRqfo5KLvmAlACQkrjEtOQ4B7wF9TdSLIs9O90MX9xA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -9071,6 +9307,13 @@ "node": ">=8" } }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/reflect.getprototypeof": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz", @@ -9399,6 +9642,20 @@ "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", "license": "MIT" }, + "node_modules/selfsigned": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-5.5.0.tgz", + "integrity": "sha512-ftnu3TW4+3eBfLRFnDEkzGxSF/10BJBkaLJuBHZX0kiPS7bRdlpZGu6YGt4KngMkdTwJE6MbjavFpqHvqVt+Ew==", + "dev": true, + "license": "MIT", + "dependencies": { + "@peculiar/x509": "^1.14.2", + "pkijs": "^3.3.3" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", @@ -10244,6 +10501,26 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsyringe": { + "version": "4.10.0", + "resolved": "https://registry.npmjs.org/tsyringe/-/tsyringe-4.10.0.tgz", + "integrity": "sha512-axr3IdNuVIxnaK5XGEUFTu3YmAQ6lllgrvqfEoR16g/HGnYY/6We4oWENtAnzK6/LpJ2ur9PAb80RBt7/U4ugw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^1.9.3" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/tsyringe/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true, + "license": "0BSD" + }, "node_modules/tunnel-rat": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/tunnel-rat/-/tunnel-rat-0.1.2.tgz", diff --git a/package.json b/package.json index 7270432..33c84f1 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "license": "MIT", "scripts": { "dev": "node server/index.js --dev", + "dev:https": "node server/index.js --dev --https", "build": "next build", "start": "node server/index.js", "lint": "eslint .", @@ -49,6 +50,7 @@ "eslint-config-next": "16.1.6", "eslint-config-prettier": "^10.1.8", "jsdom": "^27.4.0", + "selfsigned": "^5.5.0", "tailwindcss": "^4", "tw-animate-css": "^1.4.0", "typescript": "^5", diff --git a/server/index.js b/server/index.js index 06c965d..44cf7a4 100644 --- a/server/index.js +++ b/server/index.js @@ -1,4 +1,5 @@ const http = require("node:http"); +const https = require("node:https"); const next = require("next"); const { createAccessGate } = require("./access-gate"); @@ -19,8 +20,52 @@ const resolvePathname = (url) => { return (idx === -1 ? raw : raw.slice(0, idx)) || "/"; }; +const CERT_DIR = require("node:path").join(__dirname, "..", ".certs"); +const CERT_PATH = require("node:path").join(CERT_DIR, "localhost.crt"); +const KEY_PATH = require("node:path").join(CERT_DIR, "localhost.key"); + +const generateHttpsCert = async () => { + const fs = require("node:fs"); + + // Re-use a saved cert so the browser only needs to trust it once. + if (fs.existsSync(CERT_PATH) && fs.existsSync(KEY_PATH)) { + return { + key: fs.readFileSync(KEY_PATH, "utf8"), + cert: fs.readFileSync(CERT_PATH, "utf8"), + }; + } + + const selfsigned = require("selfsigned"); + const attrs = [{ name: "commonName", value: "localhost" }]; + const pems = await selfsigned.generate(attrs, { + days: 825, + keySize: 2048, + algorithm: "sha256", + extensions: [ + { + name: "subjectAltName", + altNames: [ + { type: 2, value: "localhost" }, + { type: 7, ip: "127.0.0.1" }, + ], + }, + ], + }); + + fs.mkdirSync(CERT_DIR, { recursive: true }); + fs.writeFileSync(CERT_PATH, pems.cert); + fs.writeFileSync(KEY_PATH, pems.private); + + console.info(`\nCert saved to ${CERT_DIR}`); + console.info("To make browsers trust it (macOS), run:"); + console.info(` sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain "${CERT_PATH}"\n`); + + return { key: pems.private, cert: pems.cert }; +}; + async function main() { const dev = process.argv.includes("--dev"); + const useHttps = process.argv.includes("--https") || process.env.HTTPS === "true"; const hostnames = Array.from(new Set(resolveHosts(process.env))); const hostname = hostnames[0] ?? "127.0.0.1"; const port = resolvePort(); @@ -65,11 +110,18 @@ async function main() { handleUpgrade(req, socket, head); }; + const httpsCert = useHttps ? await generateHttpsCert() : null; + const createServer = () => - http.createServer((req, res) => { - if (accessGate.handleHttp(req, res)) return; - handle(req, res); - }); + useHttps + ? https.createServer(httpsCert, (req, res) => { + if (accessGate.handleHttp(req, res)) return; + handle(req, res); + }) + : http.createServer((req, res) => { + if (accessGate.handleHttp(req, res)) return; + handle(req, res); + }); const servers = hostnames.map(() => createServer()); @@ -120,8 +172,13 @@ async function main() { ? "localhost" : hostname; - const browserUrl = `http://${hostForBrowser}:${port}`; + const protocol = useHttps ? "https" : "http"; + const browserUrl = `${protocol}://${hostForBrowser}:${port}`; console.info(`Open in browser: ${browserUrl}`); + if (useHttps) { + console.info("HTTPS mode: self-signed cert in use. You may need to accept a browser security warning once."); + console.info(`Spotify redirect URI: ${browserUrl}/office`); + } } main().catch((err) => { diff --git a/src/app/spotify/callback/page.tsx b/src/app/spotify/callback/page.tsx new file mode 100644 index 0000000..3948f28 --- /dev/null +++ b/src/app/spotify/callback/page.tsx @@ -0,0 +1,39 @@ +"use client"; + +import { useEffect } from "react"; + +export default function SpotifyCallbackPage() { + useEffect(() => { + const url = new URL(window.location.href); + const code = url.searchParams.get("code") ?? ""; + const state = url.searchParams.get("state") ?? ""; + const error = url.searchParams.get("error") ?? ""; + + if (window.opener && !window.opener.closed) { + window.opener.postMessage( + { + type: "soundclaw-spotify-auth", + code, + state, + error, + }, + "*", + ); + window.close(); + } + }, []); + + return ( +
+
+
+ Soundclaw +
+

Finishing Spotify sign-in

+

+ You can close this window if it does not close automatically. +

+
+
+ ); +} diff --git a/src/features/office/screens/OfficeScreen.tsx b/src/features/office/screens/OfficeScreen.tsx index c5a983d..9d6d92a 100644 --- a/src/features/office/screens/OfficeScreen.tsx +++ b/src/features/office/screens/OfficeScreen.tsx @@ -112,6 +112,13 @@ import { HistoryPanel } from "@/features/office/components/panels/HistoryPanel"; import { InboxPanel } from "@/features/office/components/panels/InboxPanel"; import { PlaybooksPanel } from "@/features/office/components/panels/PlaybooksPanel"; import { SkillsMarketplaceModal } from "@/features/office/components/panels/SkillsMarketplaceModal"; +import { JukeboxPanel } from "@/features/spotify-jukebox/components/JukeboxPanel"; +import { JukeboxDisabledPanel } from "@/features/spotify-jukebox/components/JukeboxDisabledPanel"; +import { executeBrowserJukeboxCommand } from "@/features/spotify-jukebox/agentBridge"; +import { + SOUNDCLAW_PLAYBACK_STARTED_EVENT_NAME, + useJukeboxStore, +} from "@/features/spotify-jukebox/store"; import { useOfficeSkillTriggers } from "@/features/office/hooks/useOfficeSkillTriggers"; import { useRemoteOfficePresence } from "@/features/office/hooks/useRemoteOfficePresence"; import { useRemoteOfficeLayout } from "@/features/office/hooks/useRemoteOfficeLayout"; @@ -147,6 +154,7 @@ import { buildOfficeDeskMonitor, type OfficeDeskMonitor, } from "@/lib/office/deskMonitor"; +import { deriveSkillReadinessState } from "@/lib/skills/presentation"; import type { StandupAgentSnapshot } from "@/lib/office/standup/types"; import type { SkillStatusEntry } from "@/lib/skills/types"; @@ -175,6 +183,31 @@ const GYM_WORKOUT_LATCH_MS = 60_000; const MAIN_AGENT_ID = "main"; const MAX_OPENCLAW_LOG_ENTRIES = 200; const MAX_OPENCLAW_AGENT_OUTPUT_LINES = 12; +const OFFICE_DANCE_MS = 60_000; + +const getLatestUserRequestForAgent = ( + agent: AgentState, +): { text: string; requestKey: string } | null => { + const transcriptEntries = Array.isArray(agent.transcriptEntries) + ? agent.transcriptEntries + : []; + for (let index = transcriptEntries.length - 1; index >= 0; index -= 1) { + const entry = transcriptEntries[index]; + if (!entry || entry.role !== "user") continue; + const text = entry.text.trim(); + if (!text) continue; + return { + text, + requestKey: `${agent.sessionKey}:${entry.sequenceKey}:${text}`, + }; + } + const fallback = agent.lastUserMessage?.trim() ?? ""; + if (!fallback) return null; + return { + text: fallback, + requestKey: `${agent.sessionKey}:fallback:${fallback}`, + }; +}; type OpenClawLogEntry = { id: string; @@ -890,12 +923,67 @@ export function OfficeScreen({ const [gatewayModels, setGatewayModels] = useState([]); const [sidebarOpen, setSidebarOpen] = useState(false); const [marketplaceOpen, setMarketplaceOpen] = useState(false); + const [danceUntilByAgentId, setDanceUntilByAgentId] = useState>({}); + const initJukeboxStore = useJukeboxStore((state) => state.init); + const jukeboxToken = useJukeboxStore((state) => state.token); + // Auto-open jukebox panel for legacy direct-auth callbacks. + const [jukeboxOpen, setJukeboxOpen] = useState(() => { + if (typeof window === "undefined") return false; + const searchParams = new URL(window.location.href).searchParams; + return searchParams.has("code"); + }); const [activeSidebarTab, setActiveSidebarTab] = useState("inbox"); + const pendingJukeboxCommandTimeoutsRef = useRef< + Map + >(new Map()); + const handledJukeboxRequestKeyByAgentIdRef = useRef>({}); const router = useRouter(); const { showOnboarding, completeOnboarding, resetOnboarding } = useOnboardingState(); const [forceShowOnboarding, setForceShowOnboarding] = useState(false); + useEffect(() => { + initJukeboxStore(); + }, [initJukeboxStore]); + useEffect(() => { + const handlePlaybackStarted = () => { + const now = Date.now(); + const until = now + OFFICE_DANCE_MS; + setDanceUntilByAgentId((previous) => { + const next: Record = {}; + for (const agent of state.agents) { + next[agent.agentId] = until; + } + return { ...previous, ...next }; + }); + }; + window.addEventListener( + SOUNDCLAW_PLAYBACK_STARTED_EVENT_NAME, + handlePlaybackStarted, + ); + return () => { + window.removeEventListener( + SOUNDCLAW_PLAYBACK_STARTED_EVENT_NAME, + handlePlaybackStarted, + ); + }; + }, [state.agents]); + useEffect(() => { + const now = Date.now(); + setDanceUntilByAgentId((previous) => + Object.fromEntries( + Object.entries(previous).filter(([, until]) => until > now), + ), + ); + }, [state.agents]); + useEffect(() => { + return () => { + for (const pendingEntry of pendingJukeboxCommandTimeoutsRef.current.values()) { + window.clearTimeout(pendingEntry.timeoutId); + } + pendingJukeboxCommandTimeoutsRef.current.clear(); + }; + }, []); const { loaded: officeTitleLoaded, title: officeTitle, @@ -2245,6 +2333,7 @@ export function OfficeScreen({ return { ...base, + danceUntilByAgentId: danceUntilByAgentId, deskHoldByAgentId: { ...base.deskHoldByAgentId, ...skillTriggerHoldMaps.deskHoldByAgentId, @@ -2257,6 +2346,10 @@ export function OfficeScreen({ ...base.gymHoldByAgentId, ...skillTriggerHoldMaps.gymHoldByAgentId, }, + jukeboxHoldByAgentId: { + ...base.jukeboxHoldByAgentId, + ...skillTriggerHoldMaps.jukeboxHoldByAgentId, + }, qaHoldByAgentId: { ...base.qaHoldByAgentId, ...skillTriggerHoldMaps.qaHoldByAgentId, @@ -2268,6 +2361,7 @@ export function OfficeScreen({ }; }, [ animationNowMs, + danceUntilByAgentId, marketplaceGymHoldByAgentId, officeTriggerState, skillTriggers.movementTargetByAgentId, @@ -2276,6 +2370,7 @@ export function OfficeScreen({ const { deskHoldByAgentId, githubHoldByAgentId, + jukeboxHoldByAgentId, manualGymUntilByAgentId, pendingStandupRequest, phoneBoothHoldByAgentId, @@ -3453,6 +3548,109 @@ export function OfficeScreen({ }) ?? null, [marketplace.skillsReport], ); + const soundclawSkill = useMemo( + () => + marketplace.skillsReport?.skills.find((skill) => { + const normalizedKey = skill.skillKey.trim().toLowerCase(); + const normalizedName = skill.name.trim().toLowerCase(); + return normalizedKey === "soundclaw" || normalizedName === "soundclaw"; + }) ?? null, + [marketplace.skillsReport], + ); + const soundclawReady = useMemo( + () => (soundclawSkill ? deriveSkillReadinessState(soundclawSkill) === "ready" : false), + [soundclawSkill] + ); + + useEffect(() => { + if (!soundclawReady || !jukeboxToken) { + return; + } + + const pending = pendingJukeboxCommandTimeoutsRef.current; + const activeAgentIds = new Set(); + + for (const agent of state.agents) { + if (skillTriggers.movementTargetByAgentId[agent.agentId] !== "jukebox") { + continue; + } + + const request = getLatestUserRequestForAgent(agent); + if (!request) { + continue; + } + + activeAgentIds.add(agent.agentId); + const handledKey = handledJukeboxRequestKeyByAgentIdRef.current[agent.agentId]; + if (handledKey === request.requestKey) { + continue; + } + + const existing = pending.get(agent.agentId); + if (existing?.requestKey === request.requestKey) { + continue; + } + if (existing) { + window.clearTimeout(existing.timeoutId); + pending.delete(agent.agentId); + } + + const timeoutId = window.setTimeout(() => { + void executeBrowserJukeboxCommand(request.text).then((result) => { + if (result.ok) { + handledJukeboxRequestKeyByAgentIdRef.current[agent.agentId] = request.requestKey; + setJukeboxOpen(true); + dispatch({ + type: "appendOutput", + agentId: agent.agentId, + line: result.reply, + transcript: { + role: "assistant", + kind: "assistant", + source: "legacy", + sessionKey: agent.sessionKey, + timestampMs: Date.now(), + confirmed: true, + }, + }); + dispatch({ + type: "updateAgent", + agentId: agent.agentId, + patch: { + latestOverride: result.reply, + latestOverrideKind: null, + latestPreview: result.reply, + lastAssistantMessageAt: Date.now(), + }, + }); + } + const latest = pendingJukeboxCommandTimeoutsRef.current.get(agent.agentId); + if (latest?.timeoutId === timeoutId) { + pendingJukeboxCommandTimeoutsRef.current.delete(agent.agentId); + } + }); + }, 1400); + + pending.set(agent.agentId, { + requestKey: request.requestKey, + timeoutId, + }); + } + + for (const [agentId, pendingEntry] of pending.entries()) { + if (activeAgentIds.has(agentId)) continue; + window.clearTimeout(pendingEntry.timeoutId); + pending.delete(agentId); + } + }, [ + jukeboxToken, + skillTriggers.movementTargetByAgentId, + soundclawReady, + state.agents, + ]); + + // No longer force-close the jukebox panel when skill is disabled; + // the panel handles the disabled state itself. if ( !agentsLoaded && @@ -3497,6 +3695,7 @@ export function OfficeScreen({ agent.status === "running" || deskHoldByAgentId[agent.agentId] || gymHoldByAgentId[agent.agentId] || + jukeboxHoldByAgentId[agent.agentId] || phoneBoothHoldByAgentId[agent.agentId] || smsBoothHoldByAgentId[agent.agentId] || qaHoldByAgentId[agent.agentId], @@ -3526,6 +3725,7 @@ export function OfficeScreen({ monitorAgentId={monitorAgentId} monitorByAgentId={monitorByAgentId} githubSkill={githubSkill} + soundclawEnabled={soundclawReady} officeTitle={officeTitle} officeTitleLoaded={officeTitleLoaded} remoteOfficeEnabled={remoteOfficeEnabled} @@ -3615,7 +3815,27 @@ export function OfficeScreen({ onOpenGithubSkillSetup={() => { setMarketplaceOpen(true); }} + onJukeboxInteract={() => { + setJukeboxOpen(true); + }} /> + {jukeboxOpen ? ( + soundclawReady ? ( + setJukeboxOpen(false)} + selectedAgentName={focusedChatAgent?.name ?? null} + /> + ) : ( + setJukeboxOpen(false)} + onInstall={() => { + setJukeboxOpen(false); + setMarketplaceOpen(true); + }} + /> + ) + ) : null} {showEmptyFleetBanner ? ( diff --git a/src/features/retro-office/RetroOffice3D.tsx b/src/features/retro-office/RetroOffice3D.tsx index 0760e7e..56e8d73 100644 --- a/src/features/retro-office/RetroOffice3D.tsx +++ b/src/features/retro-office/RetroOffice3D.tsx @@ -76,6 +76,7 @@ import { ensureOfficePingPongTable, ensureOfficeQaLab, ensureOfficeSmsBooth, + ensureOfficeJukebox, ensureOfficeServerRoom, isRetiredPingPongLamp, materializeDefaults, @@ -148,6 +149,7 @@ import type { import type { NavGrid } from "@/features/retro-office/core/navigation"; import type { OfficeLayoutSnapshot } from "@/lib/office/layoutSnapshot"; import { AgentModel as AgentObjectModel } from "@/features/retro-office/objects/agents"; +import { JukeboxModel as InteractiveJukeboxModel } from "@/features/retro-office/objects/Jukebox"; import { FurnitureModel as GenericFurnitureModel, InstancedFurnitureItems as InstancedFurnitureItemsModel, @@ -359,6 +361,7 @@ const PALETTE: PaletteEntry[] = [ { type: "fridge", label: "Fridge", icon: "🧊", defaults: { w: 40, h: 80 } }, { type: "water_cooler", label: "Water", icon: "💧", defaults: {} }, { type: "atm", label: "ATM", icon: "🏧", defaults: { facing: 270 } }, + { type: "jukebox", label: "Jukebox", icon: "🎵", defaults: { facing: 0 } }, { type: "whiteboard", label: "Whiteboard", @@ -403,11 +406,7 @@ const PALETTE: PaletteEntry[] = [ // CAMERA SETUP — sets lookAt after mount // ============================================================ -function CameraRig({ - target, -}: { - target: [number, number, number]; -}) { +function CameraRig({ target }: { target: [number, number, number] }) { const { camera } = useThree(); useEffect(() => { camera.lookAt(...target); @@ -444,8 +443,9 @@ const ReadOnlyFurnitureClone = memo(function ReadOnlyFurnitureClone({ {furniture.map((item) => - item.type === "wall" || item.type === "desk_cubicle" || item.type === "chair" ? null - : item.type === "door" ? ( + item.type === "wall" || + item.type === "desk_cubicle" || + item.type === "chair" ? null : item.type === "door" ? ( , lastSeenByAgentId: Record = {}, deskHoldByAgentId: Record = {}, + danceUntilByAgentId: Record = {}, gymHoldByAgentId: Record = {}, smsBoothHoldByAgentId: Record = {}, phoneBoothHoldByAgentId: Record = {}, @@ -882,13 +883,17 @@ function useAgentTick( ); const pickRoamPoint = useCallback((agentId: string) => { if (isRemoteOfficeAgentId(agentId)) { - return REMOTE_ROAM_POINTS[Math.floor(Math.random() * REMOTE_ROAM_POINTS.length)]; + return REMOTE_ROAM_POINTS[ + Math.floor(Math.random() * REMOTE_ROAM_POINTS.length) + ]; } return ROAM_POINTS[Math.floor(Math.random() * ROAM_POINTS.length)]; }, []); const pickSpawnPoint = useCallback((agentId: string) => { if (isRemoteOfficeAgentId(agentId)) { - return REMOTE_ROAM_POINTS[Math.floor(Math.random() * REMOTE_ROAM_POINTS.length)]; + return REMOTE_ROAM_POINTS[ + Math.floor(Math.random() * REMOTE_ROAM_POINTS.length) + ]; } return { x: Math.random() * 800 + 100, @@ -1046,11 +1051,13 @@ function useAgentTick( ? resolveMeetingTarget(agent.id) : null; const smsBoothItem = - (furnitureRef.current ?? []).find((item) => item.type === "sms_booth") ?? - null; + (furnitureRef.current ?? []).find( + (item) => item.type === "sms_booth", + ) ?? null; const phoneBoothItem = - (furnitureRef.current ?? []).find((item) => item.type === "phone_booth") ?? - null; + (furnitureRef.current ?? []).find( + (item) => item.type === "phone_booth", + ) ?? null; if (agent.status === "working" && !explicitDeskHold && deskPos) stickyUntilRef.current.set(agent.id, now + DESK_STICKY_MS); @@ -1449,19 +1456,19 @@ function useAgentTick( x: smsBoothRoute.targetX, y: smsBoothRoute.targetY, } - : explicitPhoneBoothHold - ? { - x: phoneBoothRoute.targetX, - y: phoneBoothRoute.targetY, - } - : explicitQaHold - ? { x: qaLabRoute.targetX, y: qaLabRoute.targetY } - : explicitGithubHold + : explicitPhoneBoothHold ? { - x: serverRoomRoute.targetX, - y: serverRoomRoute.targetY, + x: phoneBoothRoute.targetX, + y: phoneBoothRoute.targetY, } - : deskPos; + : explicitQaHold + ? { x: qaLabRoute.targetX, y: qaLabRoute.targetY } + : explicitGithubHold + ? { + x: serverRoomRoute.targetX, + y: serverRoomRoute.targetY, + } + : deskPos; if (!nextTarget) { ns.interactionTarget = undefined; ns.serverRoomStage = undefined; @@ -1486,13 +1493,13 @@ function useAgentTick( ? "gym" : explicitSmsBoothHold ? "sms_booth" - : explicitPhoneBoothHold - ? "phone_booth" - : explicitQaHold - ? "qa_lab" - : explicitGithubHold - ? "server_room" - : "desk"; + : explicitPhoneBoothHold + ? "phone_booth" + : explicitQaHold + ? "qa_lab" + : explicitGithubHold + ? "server_room" + : "desk"; ns.phoneBoothStage = explicitMeetingHold || explicitGymHold || @@ -1501,9 +1508,7 @@ function useAgentTick( ? undefined : phoneBoothRoute.stage; ns.smsBoothStage = - explicitMeetingHold || - explicitGymHold || - !explicitSmsBoothHold + explicitMeetingHold || explicitGymHold || !explicitSmsBoothHold ? undefined : smsBoothRoute.stage; ns.serverRoomStage = explicitMeetingHold @@ -1512,11 +1517,11 @@ function useAgentTick( ? undefined : explicitSmsBoothHold ? undefined - : explicitPhoneBoothHold - ? undefined - : explicitGithubHold - ? serverRoomRoute.stage - : undefined; + : explicitPhoneBoothHold + ? undefined + : explicitGithubHold + ? serverRoomRoute.stage + : undefined; ns.gymStage = explicitMeetingHold ? undefined : explicitGymHold @@ -1526,11 +1531,11 @@ function useAgentTick( ? undefined : explicitSmsBoothHold ? undefined - : explicitPhoneBoothHold - ? undefined - : explicitQaHold - ? qaLabRoute.stage - : undefined; + : explicitPhoneBoothHold + ? undefined + : explicitQaHold + ? qaLabRoute.stage + : undefined; ns.qaLabStationType = explicitQaHold ? qaStationPos.stationType : undefined; @@ -1600,22 +1605,22 @@ function useAgentTick( x: smsBoothRoute.targetX, y: smsBoothRoute.targetY, } - : explicitPhoneBoothHold - ? { - x: phoneBoothRoute.targetX, - y: phoneBoothRoute.targetY, - } - : explicitQaHold - ? { - x: qaLabRoute.targetX, - y: qaLabRoute.targetY, - } - : explicitGithubHold + : explicitPhoneBoothHold ? { - x: serverRoomRoute.targetX, - y: serverRoomRoute.targetY, + x: phoneBoothRoute.targetX, + y: phoneBoothRoute.targetY, } - : (deskPos ?? { x: sx, y: sy }) + : explicitQaHold + ? { + x: qaLabRoute.targetX, + y: qaLabRoute.targetY, + } + : explicitGithubHold + ? { + x: serverRoomRoute.targetX, + y: serverRoomRoute.targetY, + } + : (deskPos ?? { x: sx, y: sy }) : { x: sx, y: sy }; ns = { x: sx, @@ -1643,15 +1648,15 @@ function useAgentTick( ? "gym" : explicitSmsBoothHold ? "sms_booth" - : explicitPhoneBoothHold - ? "phone_booth" - : explicitQaHold - ? "qa_lab" - : explicitGithubHold - ? "server_room" - : deskPos - ? "desk" - : undefined, + : explicitPhoneBoothHold + ? "phone_booth" + : explicitQaHold + ? "qa_lab" + : explicitGithubHold + ? "server_room" + : deskPos + ? "desk" + : undefined, smsBoothStage: explicitMeetingHold || explicitGymHold || !explicitSmsBoothHold ? undefined @@ -2018,14 +2023,14 @@ function useAgentTick( agent.interactionTarget === "sms_booth" ? "standing" : agent.interactionTarget === "phone_booth" - ? "standing" - : agent.interactionTarget === "server_room" - ? "standing" - : agent.interactionTarget === "gym" - ? "working_out" - : agent.interactionTarget === "qa_lab" + ? "standing" + : agent.interactionTarget === "server_room" ? "standing" - : "sitting"; + : agent.interactionTarget === "gym" + ? "working_out" + : agent.interactionTarget === "qa_lab" + ? "standing" + : "sitting"; if (agent.interactionTarget === "sms_booth") { nf = agent.facing; } else if (agent.interactionTarget === "phone_booth") { @@ -2132,6 +2137,11 @@ function useAgentTick( } } + if ((danceUntilByAgentId[agent.id] ?? 0) > now && ns !== "away") { + ns = "dancing"; + npath = []; + } + return { ...agent, x: nx, @@ -2163,7 +2173,8 @@ function useAgentTick( if ("role" in mi && mi.role === "janitor") continue; if ( moved[i].state === "sitting" || - moved[i].state === "working_out" + moved[i].state === "working_out" || + moved[i].state === "dancing" ) continue; if (moved[i].pingPongUntil !== undefined && moved[i].state !== "walking") @@ -2178,7 +2189,9 @@ function useAgentTick( const bucketY = Math.floor(mi.y / collisionCellSize); for (let offsetY = -1; offsetY <= 1; offsetY += 1) { for (let offsetX = -1; offsetX <= 1; offsetX += 1) { - const bucket = collisionBuckets.get(`${bucketX + offsetX}:${bucketY + offsetY}`); + const bucket = collisionBuckets.get( + `${bucketX + offsetX}:${bucketY + offsetY}`, + ); if (!bucket) continue; for (const j of bucket) { if (i === j) continue; @@ -2246,7 +2259,13 @@ function useAgentTick( } }; - return { renderAgentsRef, renderAgentLookupRef, tick, deskByAgentRef, planPath }; + return { + renderAgentsRef, + renderAgentLookupRef, + tick, + deskByAgentRef, + planPath, + }; } // ============================================================ @@ -2256,7 +2275,9 @@ function useAgentTick( const AWAY_THRESHOLD_MS = 15 * 60 * 1000; const COMPACT_AGENT_BADGE_LIMIT = 6; -const estimatePhoneSpeechDurationMs = (text: string | null | undefined): number => { +const estimatePhoneSpeechDurationMs = ( + text: string | null | undefined, +): number => { const normalized = text?.trim() ?? ""; if (!normalized) return 5_000; const wordCount = normalized.split(/\s+/).filter(Boolean).length; @@ -2293,6 +2314,7 @@ export function RetroOffice3D({ monitorAgentId = null, monitorByAgentId = EMPTY_MONITOR_MAP, githubSkill = null, + soundclawEnabled = false, officeTitle = "Luke Headquarters", officeTitleLoaded = false, remoteOfficeEnabled = false, @@ -2340,17 +2362,20 @@ export function RetroOffice3D({ onTextMessageComplete, onQaLabDismiss, onOpenGithubSkillSetup, + onJukeboxInteract, }: { agents: OfficeAgent[]; animationState?: Pick< OfficeAnimationState, | "cleaningCues" + | "danceUntilByAgentId" | "deskHoldByAgentId" | "githubHoldByAgentId" | "gymHoldByAgentId" | "phoneBoothHoldByAgentId" | "smsBoothHoldByAgentId" | "qaHoldByAgentId" + | "jukeboxHoldByAgentId" > | null; readOnly?: boolean; storageNamespace?: string; @@ -2370,6 +2395,7 @@ export function RetroOffice3D({ monitorAgentId?: string | null; monitorByAgentId?: OfficeDeskMonitorMap; githubSkill?: SkillStatusEntry | null; + soundclawEnabled?: boolean; officeTitle?: string; officeTitleLoaded?: boolean; remoteOfficeEnabled?: boolean; @@ -2386,7 +2412,9 @@ export function RetroOffice3D({ voiceRepliesLoaded?: boolean; onOfficeTitleChange?: (title: string) => void; onRemoteOfficeEnabledChange?: (enabled: boolean) => void; - onRemoteOfficeSourceKindChange?: (kind: "presence_endpoint" | "openclaw_gateway") => void; + onRemoteOfficeSourceKindChange?: ( + kind: "presence_endpoint" | "openclaw_gateway", + ) => void; onRemoteOfficeLabelChange?: (label: string) => void; onRemoteOfficePresenceUrlChange?: (url: string) => void; onRemoteOfficeGatewayUrlChange?: (url: string) => void; @@ -2421,8 +2449,11 @@ export function RetroOffice3D({ onTextMessageComplete?: (agentId: string) => void; onQaLabDismiss?: () => void; onOpenGithubSkillSetup?: () => void; + onJukeboxInteract?: () => void; }) { const resolvedCleaningCues = animationState?.cleaningCues ?? cleaningCues; + const resolvedDanceUntilByAgentId = + animationState?.danceUntilByAgentId ?? EMPTY_NUMBER_RECORD; const resolvedDeskHoldByAgentId = animationState?.deskHoldByAgentId ?? deskHoldByAgentId; const resolvedGymHoldByAgentId = @@ -2438,20 +2469,26 @@ export function RetroOffice3D({ (githubReviewAgentId ? { [githubReviewAgentId]: true } : EMPTY_BOOLEAN_RECORD); + const resolvedJukeboxHoldByAgentId = + animationState?.jukeboxHoldByAgentId ?? EMPTY_BOOLEAN_RECORD; + const isJukeboxActive = Object.values(resolvedJukeboxHoldByAgentId).some(Boolean); + const [furniture, setFurniture] = useState(() => - ensureOfficeQaLab( - ensureOfficeGymRoom( - ensureOfficeServerRoom( - ensureOfficePhoneBooth( - ensureOfficeSmsBooth( - ensureOfficeAtm( - ensureOfficePingPongTable( - (loadFurniture(storageNamespace) ?? materializeDefaults()).filter( - (item) => !isRetiredPingPongLamp(item), + ensureOfficeJukebox( + ensureOfficeQaLab( + ensureOfficeGymRoom( + ensureOfficeServerRoom( + ensureOfficePhoneBooth( + ensureOfficeSmsBooth( + ensureOfficeAtm( + ensureOfficePingPongTable( + ( + loadFurniture(storageNamespace) ?? materializeDefaults() + ).filter((item) => !isRetiredPingPongLamp(item)), + ), ), ), ), - ), ), ), ), @@ -2473,12 +2510,12 @@ export function RetroOffice3D({ !remoteOfficeEnabled ? EMPTY_FURNITURE_ITEMS : remoteLayoutSnapshot - ? projectFurnitureIntoRemoteOfficeZone({ - furniture: remoteLayoutSnapshot.furniture, - sourceWidth: remoteLayoutSnapshot.width, - sourceHeight: remoteLayoutSnapshot.height, - }) - : defaultRemoteLayoutFurniture, + ? projectFurnitureIntoRemoteOfficeZone({ + furniture: remoteLayoutSnapshot.furniture, + sourceWidth: remoteLayoutSnapshot.width, + sourceHeight: remoteLayoutSnapshot.height, + }) + : defaultRemoteLayoutFurniture, [defaultRemoteLayoutFurniture, remoteLayoutSnapshot, remoteOfficeEnabled], ); const [editMode, setEditMode] = useState(false); @@ -2572,8 +2609,10 @@ export function RetroOffice3D({ const [monitorImmersiveReady, setMonitorImmersiveReady] = useState(false); const [activeAtmUid, setActiveAtmUid] = useState(null); const [atmImmersiveReady, setAtmImmersiveReady] = useState(false); - const [phoneBoothCommandArrived, setPhoneBoothCommandArrived] = useState(false); - const [phoneBoothImmersiveReady, setPhoneBoothImmersiveReady] = useState(false); + const [phoneBoothCommandArrived, setPhoneBoothCommandArrived] = + useState(false); + const [phoneBoothImmersiveReady, setPhoneBoothImmersiveReady] = + useState(false); const [phoneBoothDoorOpen, setPhoneBoothDoorOpen] = useState(false); const [phoneCallStep, setPhoneCallStep] = useState("dialing"); const [dialedDigits, setDialedDigits] = useState(""); @@ -2585,9 +2624,9 @@ export function RetroOffice3D({ const [typedMessageText, setTypedMessageText] = useState(""); const [activeTextKey, setActiveTextKey] = useState(null); const [textContacts, setTextContacts] = useState([]); - const [activeTextContactIndex, setActiveTextContactIndex] = useState( - null, - ); + const [activeTextContactIndex, setActiveTextContactIndex] = useState< + number | null + >(null); const [manualPhoneBoothOpen, setManualPhoneBoothOpen] = useState(false); const [manualPhoneCallScenario, setManualPhoneCallScenario] = useState(null); @@ -2598,14 +2637,17 @@ export function RetroOffice3D({ const activeTextMessageFlowKeyRef = useRef(null); const boothAudioCtxRef = useRef(null); const effectivePhoneBoothAgentIdRef = useRef(null); - const effectivePhoneCallScenarioRef = useRef(null); + const effectivePhoneCallScenarioRef = useRef( + null, + ); const phoneBoothAgentIdRef = useRef(null); const onPhoneCallSpeakRef = useRef(onPhoneCallSpeak); const onPhoneCallCompleteRef = useRef(onPhoneCallComplete); const onStandupArrivalsChangeRef = useRef(onStandupArrivalsChange); const lastStandupArrivalKeyRef = useRef(null); const effectiveSmsBoothAgentIdRef = useRef(null); - const effectiveTextMessageScenarioRef = useRef(null); + const effectiveTextMessageScenarioRef = + useRef(null); const smsBoothAgentIdRef = useRef(null); const onTextMessageCompleteRef = useRef(onTextMessageComplete); const [activeGithubTerminalUid, setActiveGithubTerminalUid] = useState< @@ -2732,8 +2774,13 @@ export function RetroOffice3D({ [agents, janitorActors], ); - const { renderAgentsRef, renderAgentLookupRef, tick, deskByAgentRef, planPath } = - useAgentTick( + const { + renderAgentsRef, + renderAgentLookupRef, + tick, + deskByAgentRef, + planPath, + } = useAgentTick( sceneAgents, deskLocations, assignedDeskIndexByAgentId, @@ -2743,13 +2790,14 @@ export function RetroOffice3D({ furnitureRef, lastSeenByAgentId, resolvedDeskHoldByAgentId, + resolvedDanceUntilByAgentId, resolvedGymHoldByAgentId, resolvedSmsBoothHoldByAgentId, resolvedPhoneBoothHoldByAgentId, resolvedQaHoldByAgentId, resolvedGithubReviewByAgentId, standupMeeting, - ); + ); useEffect(() => { const syncRenderAgentUi = () => { const next: Record = {}; @@ -2773,44 +2821,60 @@ export function RetroOffice3D({ : null; const agentStatusLookup = useMemo( () => - agents.reduce>((acc, agent) => { - const renderAgent = renderAgentUiById[agent.id]; - acc[agent.id] = { - isError: renderAgent?.status === "error" || agent.status === "error", - working: - renderAgent?.state === "sitting" || - renderAgent?.status === "working" || - agent.status === "working", - }; - return acc; - }, {}), + agents.reduce>( + (acc, agent) => { + const renderAgent = renderAgentUiById[agent.id]; + acc[agent.id] = { + isError: + renderAgent?.status === "error" || agent.status === "error", + working: + renderAgent?.state === "sitting" || + renderAgent?.state === "dancing" || + renderAgent?.status === "working" || + agent.status === "working", + }; + return acc; + }, + {}, + ), [agents, renderAgentUiById], ); const hoveredAgent = useMemo( - () => (hoveredAgentId ? agents.find((agent) => agent.id === hoveredAgentId) ?? null : null), + () => + hoveredAgentId + ? (agents.find((agent) => agent.id === hoveredAgentId) ?? null) + : null, [agents, hoveredAgentId], ); - const hoveredAgentStatus = hoveredAgentId ? agentStatusLookup[hoveredAgentId] ?? null : null; + const hoveredAgentStatus = hoveredAgentId + ? (agentStatusLookup[hoveredAgentId] ?? null) + : null; const handleAgentHover = useCallback((agentId: string) => { setHoveredAgentId(agentId); }, []); const handleAgentUnhover = useCallback(() => { setHoveredAgentId(null); }, []); - const handleAgentClick = useCallback((agentId: string) => { - const agent = renderAgentLookupRef.current.get(agentId); - if (!agent || !orbitRef.current) return; - const [wx, , wz] = toWorld(agent.x, agent.y); - orbitRef.current.target.set(wx, 0, wz); - orbitRef.current.update(); - if (isRemoteOfficeAgentId(agentId)) { - onAgentChatSelect?.(agentId); - } - }, [onAgentChatSelect, renderAgentLookupRef]); - const handleAgentContextMenu = useCallback((agentId: string, x: number, y: number) => { - if (isRemoteOfficeAgentId(agentId)) return; - setContextMenu({ id: agentId, x, y }); - }, []); + const handleAgentClick = useCallback( + (agentId: string) => { + const agent = renderAgentLookupRef.current.get(agentId); + if (!agent || !orbitRef.current) return; + const [wx, , wz] = toWorld(agent.x, agent.y); + orbitRef.current.target.set(wx, 0, wz); + orbitRef.current.update(); + if (isRemoteOfficeAgentId(agentId)) { + onAgentChatSelect?.(agentId); + } + }, + [onAgentChatSelect, renderAgentLookupRef], + ); + const handleAgentContextMenu = useCallback( + (agentId: string, x: number, y: number) => { + if (isRemoteOfficeAgentId(agentId)) return; + setContextMenu({ id: agentId, x, y }); + }, + [], + ); const monitorImmersive = Boolean(activeMonitor && monitorImmersiveReady); const serverTerminal = useMemo( () => furniture.find((item) => item.type === "server_terminal") ?? null, @@ -2847,11 +2911,14 @@ export function RetroOffice3D({ [furniture], ); const effectivePhoneCallScenario = - phoneCallScenario ?? (manualPhoneBoothOpen ? manualPhoneCallScenario : null); + phoneCallScenario ?? + (manualPhoneBoothOpen ? manualPhoneCallScenario : null); const effectivePhoneBoothAgentId = - phoneBoothAgentId ?? (manualPhoneBoothOpen ? "__manual_phone_booth__" : null); + phoneBoothAgentId ?? + (manualPhoneBoothOpen ? "__manual_phone_booth__" : null); const phoneBoothViewActive = - manualPhoneBoothOpen || Boolean(phoneBoothAgentId && phoneBoothCommandArrived); + manualPhoneBoothOpen || + Boolean(phoneBoothAgentId && phoneBoothCommandArrived); const activePhoneCallFlowKey = useMemo(() => { if (!effectivePhoneBoothAgentId || !effectivePhoneCallScenario) return null; return [ @@ -2863,13 +2930,14 @@ export function RetroOffice3D({ }, [effectivePhoneBoothAgentId, effectivePhoneCallScenario]); const phoneBoothImmersive = Boolean( activePhoneBooth && - effectivePhoneBoothAgentId && - effectivePhoneCallScenario && - phoneBoothViewActive && - phoneBoothImmersiveReady, + effectivePhoneBoothAgentId && + effectivePhoneCallScenario && + phoneBoothViewActive && + phoneBoothImmersiveReady, ); const effectiveTextMessageScenario = - textMessageScenario ?? (manualSmsBoothOpen ? manualTextMessageScenario : null); + textMessageScenario ?? + (manualSmsBoothOpen ? manualTextMessageScenario : null); const effectiveSmsBoothAgentId = smsBoothAgentId ?? (manualSmsBoothOpen ? "__manual_sms_booth__" : null); const smsBoothViewActive = @@ -2885,10 +2953,10 @@ export function RetroOffice3D({ }, [effectiveSmsBoothAgentId, effectiveTextMessageScenario]); const smsBoothImmersive = Boolean( activeSmsBooth && - effectiveSmsBoothAgentId && - effectiveTextMessageScenario && - smsBoothViewActive && - smsBoothImmersiveReady, + effectiveSmsBoothAgentId && + effectiveTextMessageScenario && + smsBoothViewActive && + smsBoothImmersiveReady, ); const meetingTable = useMemo( () => @@ -2949,7 +3017,10 @@ export function RetroOffice3D({ () => agents.slice(0, COMPACT_AGENT_BADGE_LIMIT), [agents], ); - const hiddenAgentCount = Math.max(0, agents.length - compactRosterAgents.length); + const hiddenAgentCount = Math.max( + 0, + agents.length - compactRosterAgents.length, + ); const standupActive = standupMeeting?.phase === "gathering" || standupMeeting?.phase === "in_progress"; @@ -3196,7 +3267,11 @@ export function RetroOffice3D({ }, [getBoothAudioContext]); const playTextKeyTone = useCallback( - async (options?: { frequency?: number; durationMs?: number; gain?: number }) => { + async (options?: { + frequency?: number; + durationMs?: number; + gain?: number; + }) => { const audioContext = await getBoothAudioContext(); if (!audioContext) return; const now = audioContext.currentTime; @@ -3271,7 +3346,9 @@ export function RetroOffice3D({ await audioContext.resume(); } const arrayBuffer = await blob.arrayBuffer(); - const decoded = await audioContext.decodeAudioData(arrayBuffer.slice(0)); + const decoded = await audioContext.decodeAudioData( + arrayBuffer.slice(0), + ); const source = audioContext.createBufferSource(); source.buffer = decoded; source.connect(audioContext.destination); @@ -3437,10 +3514,10 @@ export function RetroOffice3D({ const agent = agentLookup.get(githubReviewAgentId); const arrived = Boolean( agent && - Math.hypot( - agent.x - SERVER_ROOM_TARGET.x, - agent.y - SERVER_ROOM_TARGET.y, - ) < 16, + Math.hypot( + agent.x - SERVER_ROOM_TARGET.x, + agent.y - SERVER_ROOM_TARGET.y, + ) < 16, ); setGithubCommandArrived((current) => current === arrived ? current : arrived, @@ -3453,11 +3530,13 @@ export function RetroOffice3D({ const agent = agentLookup.get(qaTestingAgentId); const arrived = Boolean( agent && - agent.interactionTarget === "qa_lab" && - agent.qaLabStage === "station" && - Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 16, + agent.interactionTarget === "qa_lab" && + agent.qaLabStage === "station" && + Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 16, + ); + setQaCommandArrived((current) => + current === arrived ? current : arrived, ); - setQaCommandArrived((current) => (current === arrived ? current : arrived)); } if (!phoneBoothAgentId) { @@ -3469,15 +3548,15 @@ export function RetroOffice3D({ const agent = agentLookup.get(phoneBoothAgentId); const arrived = Boolean( agent && - agent.interactionTarget === "phone_booth" && - agent.phoneBoothStage === "receiver" && - Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 16, + agent.interactionTarget === "phone_booth" && + agent.phoneBoothStage === "receiver" && + Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 16, ); const doorOpen = Boolean( agent && - agent.interactionTarget === "phone_booth" && - agent.phoneBoothStage !== undefined && - agent.phoneBoothStage !== "door_outer", + agent.interactionTarget === "phone_booth" && + agent.phoneBoothStage !== undefined && + agent.phoneBoothStage !== "door_outer", ); setPhoneBoothCommandArrived((current) => current === arrived ? current : arrived, @@ -3496,20 +3575,22 @@ export function RetroOffice3D({ const agent = agentLookup.get(smsBoothAgentId); const arrived = Boolean( agent && - agent.interactionTarget === "sms_booth" && - agent.smsBoothStage === "typing" && - Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 16, + agent.interactionTarget === "sms_booth" && + agent.smsBoothStage === "typing" && + Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 16, ); const doorOpen = Boolean( agent && - agent.interactionTarget === "sms_booth" && - agent.smsBoothStage !== undefined && - agent.smsBoothStage !== "door_outer", + agent.interactionTarget === "sms_booth" && + agent.smsBoothStage !== undefined && + agent.smsBoothStage !== "door_outer", ); setSmsBoothCommandArrived((current) => current === arrived ? current : arrived, ); - setSmsBoothDoorOpen((current) => (current === doorOpen ? current : doorOpen)); + setSmsBoothDoorOpen((current) => + current === doorOpen ? current : doorOpen, + ); } if (!standupActive || !standupMeeting) { @@ -3520,11 +3601,16 @@ export function RetroOffice3D({ return; } - const arrivedParticipants = standupMeeting.participantOrder.filter((agentId) => { - const agent = agentLookup.get(agentId); - if (!agent || agent.interactionTarget !== "meeting_room") return false; - return Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 18; - }); + const arrivedParticipants = standupMeeting.participantOrder.filter( + (agentId) => { + const agent = agentLookup.get(agentId); + if (!agent || agent.interactionTarget !== "meeting_room") + return false; + return ( + Math.hypot(agent.x - agent.targetX, agent.y - agent.targetY) < 18 + ); + }, + ); const nextArrivalsKey = arrivedParticipants.join("|"); if (lastStandupArrivalKeyRef.current === nextArrivalsKey) return; lastStandupArrivalKeyRef.current = nextArrivalsKey; @@ -3667,7 +3753,10 @@ export function RetroOffice3D({ if (pressedKey) { pulseKeyboardKey(pressedKey); } - if (index >= (scenario.messageText?.length ?? 0) && typingTimer !== null) { + if ( + index >= (scenario.messageText?.length ?? 0) && + typingTimer !== null + ) { window.clearInterval(typingTimer); typingTimer = null; clearActiveKey(); @@ -3758,7 +3847,12 @@ export function RetroOffice3D({ zoom: 228, }; prevSmsBoothViewRef.current = activeViewKey; - }, [activeSmsBooth, manualSmsBoothOpen, smsBoothAgentId, smsBoothCommandArrived]); + }, [ + activeSmsBooth, + manualSmsBoothOpen, + smsBoothAgentId, + smsBoothCommandArrived, + ]); useEffect(() => { const resetTimer = window.setTimeout(() => { @@ -3842,7 +3936,9 @@ export function RetroOffice3D({ setPhoneCallStep("complete"); stageTimer = window.setTimeout(() => { if (phoneBoothAgentIdRef.current) { - onPhoneCallCompleteRef.current?.(phoneBoothAgentIdRef.current); + onPhoneCallCompleteRef.current?.( + phoneBoothAgentIdRef.current, + ); } else { closeManualPhoneBoothView(); } @@ -3876,12 +3972,11 @@ export function RetroOffice3D({ ]); useEffect(() => { - const activeViewKey = - manualPhoneBoothOpen - ? "manual" - : phoneBoothAgentId && phoneBoothCommandArrived - ? `agent:${phoneBoothAgentId}` - : null; + const activeViewKey = manualPhoneBoothOpen + ? "manual" + : phoneBoothAgentId && phoneBoothCommandArrived + ? `agent:${phoneBoothAgentId}` + : null; if (!activeViewKey && prevPhoneBoothViewRef.current) { cameraPresetRef.current = CAMERA_PRESET_MAP.overview; } @@ -3900,7 +3995,12 @@ export function RetroOffice3D({ zoom: 210, }; prevPhoneBoothViewRef.current = activeViewKey; - }, [activePhoneBooth, manualPhoneBoothOpen, phoneBoothAgentId, phoneBoothCommandArrived]); + }, [ + activePhoneBooth, + manualPhoneBoothOpen, + phoneBoothAgentId, + phoneBoothCommandArrived, + ]); useEffect(() => { const resetTimer = window.setTimeout(() => { @@ -4850,10 +4950,13 @@ export function RetroOffice3D({ // Camera constants. const CAM_POS = DISTRICT_CAMERA_POSITION; const LOCAL_CAMERA_TARGET = useMemo( - () => toWorld(LOCAL_OFFICE_CANVAS_WIDTH / 2, LOCAL_OFFICE_CANVAS_HEIGHT / 2), + () => + toWorld(LOCAL_OFFICE_CANVAS_WIDTH / 2, LOCAL_OFFICE_CANVAS_HEIGHT / 2), [], ); - const cameraTarget = remoteOfficeEnabled ? DISTRICT_CAMERA_TARGET : LOCAL_CAMERA_TARGET; + const cameraTarget = remoteOfficeEnabled + ? DISTRICT_CAMERA_TARGET + : LOCAL_CAMERA_TARGET; const cameraZoom = remoteOfficeEnabled ? DISTRICT_CAMERA_ZOOM : 56; return ( @@ -4883,7 +4986,12 @@ export function RetroOffice3D({ - {/* Ensure camera looks at origin after mount. */} - - + {/* Ensure camera looks at origin after mount. */} + + - {/* Orbit / pan / zoom controls — disabled while follow cam is active or while editing furniture. */} - - - {/* Game loop — no React state, pure ref mutations. */} - - - {/* New Idea 2: Camera preset animator. */} - - - {/* Follow cam: third-person perspective camera trailing the selected agent. */} - - - {/* E3 Idea 3: Spotlight effect on agent chip click. */} - - - {/* Keep office lighting static to avoid extra scene churn from ambience effects. */} - - - - - {/* Floor + walls — always visible, no async loading. */} - - - {/* Wall pictures — procedural, no async loading. */} - - - {/* Environment lighting — async, wrapped in its own Suspense so floor stays visible. */} - - - - - {/* Furniture models — each loads its GLB asynchronously. */} - - {!editMode ? ( - - ) : null} - {!editMode ? ( - - ) : null} - {!editMode ? ( - - ) : null} - {furniture.map((item) => - item.type === "wall" ? ( - editMode ? ( - - ) : null - ) : item.type === "desk_cubicle" ? ( - editMode ? ( - - ) : null - ) : item.type === "chair" ? ( - editMode ? ( - - ) : null - ) : item.type === "door" ? ( - - ) : item.type === "round_table" ? ( - - ) : item.type === "keyboard" ? ( - - ) : item.type === "mouse" ? ( - - ) : item.type === "trash" ? ( - - ) : item.type === "mug" ? ( - - ) : item.type === "clock" ? ( - - ) : item.type === "atm" ? ( - - ) : item.type === "sms_booth" ? ( - - ) : item.type === "phone_booth" ? ( - - ) : item.type === "server_rack" ? ( - - ) : item.type === "server_terminal" ? ( - - ) : item.type === "vending" ? ( - - ) : item.type === "sink" ? ( - - ) : item.type === "dishwasher" ? ( - - ) : item.type === "pingpong" ? ( - - ) : item.type === "qa_terminal" ? ( - - ) : item.type === "device_rack" ? ( - - ) : item.type === "test_bench" ? ( - - ) : item.type === "treadmill" ? ( - - ) : item.type === "weight_bench" ? ( - - ) : item.type === "dumbbell_rack" ? ( - - ) : item.type === "exercise_bike" ? ( - - ) : item.type === "rowing_machine" ? ( - - ) : item.type === "kettlebell_rack" ? ( - - ) : item.type === "punching_bag" ? ( - - ) : item.type === "yoga_mat" ? ( - - ) : item.type === "stove" ? ( - - ) : item.type === "microwave" ? ( - - ) : item.type === "wall_cabinet" ? ( - - ) : ( - - ), - )} - - - {remoteLayoutFurniture.length > 0 ? ( - - ) : null} - - {/* Agents — purely imperative, driven by renderAgentsRef inside useFrame. */} - {sceneAgents.map((agent) => { - const isJanitor = "role" in agent && agent.role === "janitor"; - return ( - - ); - })} - - - - {/* Idea 7: Desk nameplates — small labels showing assigned agent above each desk. */} - - - {/* New Idea 5: Agent color trails while walking. */} - {trailMode ? ( - - ) : null} - - {/* New Idea 7: Heatmap overlay when heatmap mode is active. */} - {heatmapMode ? ( - - ) : null} - - {/* Placement ghost. */} - {editMode && - drag.kind === "placing" && - drag.itemType !== "wall" && - ghostPos && ( - - - - )} - {editMode && - drag.kind === "placing" && - drag.itemType === "wall" && - wallGhostItem ? ( - {}} - onPointerOver={() => {}} - onPointerOut={() => {}} - /> - ) : null} - {editMode && - drag.kind === "placing" && - drag.itemType === "door" && - ghostPos ? ( - {}} - onPointerOver={() => {}} - onPointerOut={() => {}} /> - ) : null} - {/* Floor raycaster for edit-mode interaction. */} - + {/* Game loop — no React state, pure ref mutations. */} + + + {/* New Idea 2: Camera preset animator. */} + + + {/* Follow cam: third-person perspective camera trailing the selected agent. */} + + + {/* E3 Idea 3: Spotlight effect on agent chip click. */} + + + {/* Keep office lighting static to avoid extra scene churn from ambience effects. */} + + + + + {/* Floor + walls — always visible, no async loading. */} + + + {/* Wall pictures — procedural, no async loading. */} + + + {/* Environment lighting — async, wrapped in its own Suspense so floor stays visible. */} + + + + + {/* Furniture models — each loads its GLB asynchronously. */} + + {!editMode ? ( + + ) : null} + {!editMode ? ( + + ) : null} + {!editMode ? ( + + ) : null} + {furniture.map((item) => + item.type === "wall" ? ( + editMode ? ( + + ) : null + ) : item.type === "desk_cubicle" ? ( + editMode ? ( + + ) : null + ) : item.type === "chair" ? ( + editMode ? ( + + ) : null + ) : item.type === "door" ? ( + + ) : item.type === "round_table" ? ( + + ) : item.type === "keyboard" ? ( + + ) : item.type === "mouse" ? ( + + ) : item.type === "trash" ? ( + + ) : item.type === "mug" ? ( + + ) : item.type === "clock" ? ( + + ) : item.type === "atm" ? ( + + ) : item.type === "jukebox" ? ( + onJukeboxInteract?.()} + /> + ) : item.type === "sms_booth" ? ( + + ) : item.type === "phone_booth" ? ( + + ) : item.type === "server_rack" ? ( + + ) : item.type === "server_terminal" ? ( + + ) : item.type === "vending" ? ( + + ) : item.type === "sink" ? ( + + ) : item.type === "dishwasher" ? ( + + ) : item.type === "pingpong" ? ( + + ) : item.type === "qa_terminal" ? ( + + ) : item.type === "device_rack" ? ( + + ) : item.type === "test_bench" ? ( + + ) : item.type === "treadmill" ? ( + + ) : item.type === "weight_bench" ? ( + + ) : item.type === "dumbbell_rack" ? ( + + ) : item.type === "exercise_bike" ? ( + + ) : item.type === "rowing_machine" ? ( + + ) : item.type === "kettlebell_rack" ? ( + + ) : item.type === "punching_bag" ? ( + + ) : item.type === "yoga_mat" ? ( + + ) : item.type === "stove" ? ( + + ) : item.type === "microwave" ? ( + + ) : item.type === "wall_cabinet" ? ( + + ) : ( + + ), + )} + + + {remoteLayoutFurniture.length > 0 ? ( + + ) : null} + + {/* Removed standalone Jukebox as it's now in the furniture loop */} + + {/* Agents — purely imperative, driven by renderAgentsRef inside useFrame. */} + {sceneAgents.map((agent) => { + const isJanitor = "role" in agent && agent.role === "janitor"; + return ( + + ); + })} + + + + {/* Idea 7: Desk nameplates — small labels showing assigned agent above each desk. */} + + + {/* New Idea 5: Agent color trails while walking. */} + {trailMode ? ( + + ) : null} + + {/* New Idea 7: Heatmap overlay when heatmap mode is active. */} + {heatmapMode ? ( + + ) : null} + + {/* Placement ghost. */} + {editMode && + drag.kind === "placing" && + drag.itemType !== "wall" && + ghostPos && ( + + + + )} + {editMode && + drag.kind === "placing" && + drag.itemType === "wall" && + wallGhostItem ? ( + {}} + onPointerOver={() => {}} + onPointerOut={() => {}} + /> + ) : null} + {editMode && + drag.kind === "placing" && + drag.itemType === "door" && + ghostPos ? ( + {}} + onPointerOver={() => {}} + onPointerOut={() => {}} + /> + ) : null} + + {/* Floor raycaster for edit-mode interaction. */} + ) : null} @@ -5511,7 +5642,11 @@ export function RetroOffice3D({ icon: , title: "Front desk", }, - { key: "lounge", icon: , title: "Lounge" }, + { + key: "lounge", + icon: , + title: "Lounge", + }, ] as const ).map(({ key, icon, title }) => ( + + + + + ); +} diff --git a/src/features/spotify-jukebox/components/JukeboxPanel.tsx b/src/features/spotify-jukebox/components/JukeboxPanel.tsx new file mode 100644 index 0000000..c084fa9 --- /dev/null +++ b/src/features/spotify-jukebox/components/JukeboxPanel.tsx @@ -0,0 +1,463 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { useJukeboxStore } from "../store"; +import { + startSpotifyAuth, + buildRedirectUri, + loadToken, + exchangeCodeForToken, + loadCallbackBaseUrl, + saveCallbackBaseUrl, + loadAuthState, +} from "../auth"; +import type { SpotifyTrack } from "../spotifyApi"; + +type JukeboxPanelProps = { + onClose: () => void; + selectedAgentName?: string | null; + client?: unknown; +}; + +// --------------------------------------------------------------------------- +// Root panel +// --------------------------------------------------------------------------- + +export function JukeboxPanel({ onClose }: JukeboxPanelProps) { + const { view, init } = useJukeboxStore(); + + useEffect(() => { + init(); + const handleMessage = (event: MessageEvent) => { + const callbackBaseUrl = loadCallbackBaseUrl(); + if (!callbackBaseUrl) return; + const callbackOrigin = new URL(callbackBaseUrl).origin; + if (event.origin !== callbackOrigin) return; + const payload = event.data as + | { + type?: string; + code?: string; + error?: string; + state?: string; + } + | undefined; + if (!payload || payload.type !== "soundclaw-spotify-auth") return; + if (payload.error) return; + if (!payload.code) return; + if (payload.state !== loadAuthState()) return; + + const { clientId, setToken } = useJukeboxStore.getState(); + void exchangeCodeForToken(payload.code, clientId, buildRedirectUri(callbackBaseUrl)).then( + (ok) => { + if (!ok) return; + const token = loadToken(); + if (token) setToken(token); + }, + ); + }; + + window.addEventListener("message", handleMessage); + return () => window.removeEventListener("message", handleMessage); + }, []); + + return ( +
+
+ {/* Header. */} +
+
+ 🎵 +
+
+ Soundclaw +
+

Office Jukebox

+
+
+ +
+ + {/* Body. */} +
+ {view === "setup" ? : } +
+
+
+ ); +} + +// --------------------------------------------------------------------------- +// Setup view — shown before the user authenticates +// --------------------------------------------------------------------------- + +function SetupView() { + const { clientId, setClientId } = useJukeboxStore(); + const [inputId, setInputId] = useState(clientId); + const [callbackBaseUrl, setCallbackBaseUrl] = useState(() => loadCallbackBaseUrl()); + const [isRedirecting, setIsRedirecting] = useState(false); + const redirectUri = buildRedirectUri(callbackBaseUrl); + const localhostOrigin = + typeof window !== "undefined" ? `${window.location.protocol}//${window.location.host}` : ""; + const callbackLooksValid = /^https:\/\/.+/i.test(callbackBaseUrl.trim()); + + const handleConnect = async () => { + if (!inputId.trim() || !redirectUri) return; + saveCallbackBaseUrl(callbackBaseUrl); + setClientId(inputId.trim()); + setIsRedirecting(true); + const popup = window.open( + "", + "soundclaw-spotify-auth", + "popup=yes,width=520,height=760,resizable=yes,scrollbars=yes", + ); + if (!popup) { + setIsRedirecting(false); + return; + } + popup.document.write("

Redirecting to Spotify...

"); + await startSpotifyAuth(inputId.trim(), redirectUri, popup); + setIsRedirecting(false); + }; + + return ( +
+
+ Keep Claw3D open on {localhostOrigin}. + Spotify will redirect to your ngrok callback, which sends the auth code back into this window. +
+ + {!callbackLooksValid && callbackBaseUrl.trim().length > 0 && ( +
+ Enter a valid HTTPS ngrok URL, for example https://your-id.ngrok-free.app. +
+ )} + + {/* What you need card. */} +
+

+ ⚠️ What you need before connecting +

+
    +
  1. + 1 + + Go to{" "} + + developer.spotify.com/dashboard + {" "} + and create an app (or use an existing one). + +
    2. +
  2. + 2 + + In your Spotify app settings, add this Redirect URI: + +
    3. + {redirectUri && ( +
  3. + + {redirectUri} + + +
    4. + )} +
  4. + 3 + Paste your public ngrok URL below, then use the exact redirect shown here in Spotify. +
    5. +
  5. + 4 + Keep this local office tab open while authenticating. The popup callback will hand the code back to this page. +
    6. +
  6. + 5 + Make sure Spotify is open and playing on at least one device before using playback controls. +
    7. +
+
+ +
+ + setCallbackBaseUrl(e.target.value)} + placeholder="https://your-id.ngrok-free.app" + className="w-full rounded-xl border border-white/10 bg-slate-900 px-4 py-2.5 font-mono text-sm text-white placeholder-slate-600 focus:border-cyan-500/50 focus:outline-none focus:ring-1 focus:ring-cyan-500/30" + /> +

+ This is only used for the Spotify OAuth callback bridge. Your app can stay on {localhostOrigin}. +

+
+ + {/* Client ID input. */} +
+ + setInputId(e.target.value)} + placeholder="e.g. 1a2b3c4d5e6f…" + className="w-full rounded-xl border border-white/10 bg-slate-900 px-4 py-2.5 font-mono text-sm text-white placeholder-slate-600 focus:border-cyan-500/50 focus:outline-none focus:ring-1 focus:ring-cyan-500/30" + /> +

+ Stored locally in your browser. Never sent to any server other than Spotify. +

+
+ + +
+ ); +} + +// --------------------------------------------------------------------------- +// Player view — shown after authentication +// --------------------------------------------------------------------------- + +function PlayerView() { + const { + playerState, + searchResults, + searchQuery, + isSearching, + isLoadingPlayer, + error, + refreshPlayer, + search, + setSearchQuery, + play, + pause, + resume, + next, + previous, + volume, + disconnect, + } = useJukeboxStore(); + + const searchDebounce = useRef | null>(null); + + // Poll player state every 5 seconds. + useEffect(() => { + refreshPlayer(); + const id = window.setInterval(() => { void refreshPlayer(); }, 5000); + return () => window.clearInterval(id); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const handleSearchChange = (value: string) => { + setSearchQuery(value); + if (searchDebounce.current) clearTimeout(searchDebounce.current); + searchDebounce.current = setTimeout(() => { + if (value.trim()) void search(value); + }, 400); + }; + + const track = playerState?.track; + const albumArt = track?.album.images[0]?.url ?? null; + + return ( +
+ {error && ( +
+ {error} +
+ )} + + {/* Now playing. */} +
+
+ Now playing +
+ {isLoadingPlayer && !track ? ( +
+
+ Loading player… +
+ ) : track ? ( +
+ {albumArt && ( + // eslint-disable-next-line @next/next/no-img-element + {track.album.name} + )} +
+
{track.name}
+
+ {track.artists.map((a) => a.name).join(", ")} +
+
{track.album.name}
+
+
+ ) : ( +

+ No active playback. Open Spotify on a device first, then hit play. +

+ )} + + {/* Transport controls. */} +
+ void previous()} title="Previous" /> + {playerState?.isPlaying ? ( + void pause()} title="Pause" large /> + ) : ( + void resume()} title="Play" large /> + )} + void next()} title="Next" /> +
+ + {/* Volume. */} + {playerState && ( +
+ 🔈 + void volume(Number(e.target.value))} + className="h-1.5 w-full cursor-pointer accent-cyan-400" + /> + + {playerState.volumePercent}% + +
+ )} +
+ + {/* Search. */} +
+
+ Search tracks +
+
+ handleSearchChange(e.target.value)} + placeholder="Artist, song, or album…" + className="w-full rounded-xl border border-white/10 bg-slate-900 py-2.5 pl-4 pr-10 text-sm text-white placeholder-slate-600 focus:border-cyan-500/50 focus:outline-none focus:ring-1 focus:ring-cyan-500/30" + /> + {isSearching && ( +
+
+
+ )} +
+ + {searchResults.length > 0 && ( +
    + {searchResults.map((track) => ( + void play(track.uri)} /> + ))} +
+ )} +
+ + {/* Disconnect. */} +
+ +
+
+ ); +} + +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +function ControlButton({ + icon, + onClick, + title, + large, +}: { + icon: string; + onClick: () => void; + title: string; + large?: boolean; +}) { + return ( + + ); +} + +function SearchResult({ + track, + onPlay, +}: { + track: SpotifyTrack; + onPlay: () => void; +}) { + const art = track.album.images[track.album.images.length - 1]?.url ?? null; + return ( +
  • + {art && ( + // eslint-disable-next-line @next/next/no-img-element + {track.album.name} + )} +
    +
    {track.name}
    +
    + {track.artists.map((a) => a.name).join(", ")} · {track.album.name} +
    +
    + +
    • + ); +} diff --git a/src/features/spotify-jukebox/spotifyApi.ts b/src/features/spotify-jukebox/spotifyApi.ts new file mode 100644 index 0000000..b697f83 --- /dev/null +++ b/src/features/spotify-jukebox/spotifyApi.ts @@ -0,0 +1,115 @@ +"use client"; + +// Thin Spotify Web API wrapper used by the jukebox panel. + +export type SpotifyTrack = { + id: string; + name: string; + uri: string; + durationMs: number; + artists: { name: string }[]; + album: { name: string; images: { url: string; width: number; height: number }[] }; +}; + +export type PlayerState = { + isPlaying: boolean; + progressMs: number; + track: SpotifyTrack | null; + volumePercent: number; + deviceId: string | null; +}; + +type RawTrack = { + id: string; + name: string; + uri: string; + duration_ms: number; + artists: { name: string }[]; + album: { name: string; images: { url: string; width: number; height: number }[] }; +}; + +type RawPlayerState = { + is_playing: boolean; + progress_ms: number; + device: { id: string; volume_percent: number }; + item: RawTrack | null; +}; + +const BASE = "https://api.spotify.com/v1"; + +const headers = (token: string) => ({ + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", +}); + +const mapTrack = (raw: RawTrack): SpotifyTrack => ({ + id: raw.id, + name: raw.name, + uri: raw.uri, + durationMs: raw.duration_ms, + artists: raw.artists, + album: raw.album, +}); + +// --------------------------------------------------------------------------- +// Player state +// --------------------------------------------------------------------------- + +export const fetchPlayerState = async (token: string): Promise => { + const res = await fetch(`${BASE}/me/player`, { headers: headers(token) }); + if (res.status === 204 || !res.ok) return null; + const data = (await res.json()) as RawPlayerState; + return { + isPlaying: data.is_playing, + progressMs: data.progress_ms, + track: data.item ? mapTrack(data.item) : null, + volumePercent: data.device?.volume_percent ?? 50, + deviceId: data.device?.id ?? null, + }; +}; + +// --------------------------------------------------------------------------- +// Search +// --------------------------------------------------------------------------- + +export const searchTracks = async (token: string, query: string): Promise => { + const params = new URLSearchParams({ q: query, type: "track", limit: "10" }); + const res = await fetch(`${BASE}/search?${params}`, { headers: headers(token) }); + if (!res.ok) return []; + const data = await res.json() as { tracks: { items: RawTrack[] } }; + return (data.tracks?.items ?? []).map(mapTrack); +}; + +// --------------------------------------------------------------------------- +// Playback controls +// --------------------------------------------------------------------------- + +export const playTrack = async (token: string, uri: string, deviceId?: string | null): Promise => { + const params = deviceId ? `?device_id=${deviceId}` : ""; + await fetch(`${BASE}/me/player/play${params}`, { + method: "PUT", + headers: headers(token), + body: JSON.stringify({ uris: [uri] }), + }); +}; + +export const pausePlayback = async (token: string): Promise => { + await fetch(`${BASE}/me/player/pause`, { method: "PUT", headers: headers(token) }); +}; + +export const resumePlayback = async (token: string): Promise => { + await fetch(`${BASE}/me/player/play`, { method: "PUT", headers: headers(token) }); +}; + +export const skipToNext = async (token: string): Promise => { + await fetch(`${BASE}/me/player/next`, { method: "POST", headers: headers(token) }); +}; + +export const skipToPrevious = async (token: string): Promise => { + await fetch(`${BASE}/me/player/previous`, { method: "POST", headers: headers(token) }); +}; + +export const setVolume = async (token: string, volumePercent: number): Promise => { + const params = new URLSearchParams({ volume_percent: String(Math.round(volumePercent)) }); + await fetch(`${BASE}/me/player/volume?${params}`, { method: "PUT", headers: headers(token) }); +}; diff --git a/src/features/spotify-jukebox/store.ts b/src/features/spotify-jukebox/store.ts new file mode 100644 index 0000000..67cf2ed --- /dev/null +++ b/src/features/spotify-jukebox/store.ts @@ -0,0 +1,182 @@ +"use client"; + +import { create } from "zustand"; +import { loadToken, clearToken, loadClientId, saveClientId } from "./auth"; +import { + fetchPlayerState, + searchTracks, + playTrack, + pausePlayback, + resumePlayback, + skipToNext, + skipToPrevious, + setVolume, + type PlayerState, + type SpotifyTrack, +} from "./spotifyApi"; + +const SOUNDCLAW_PLAYBACK_STARTED_EVENT = "soundclaw:playback-started"; + +const emitPlaybackStarted = () => { + if (typeof window === "undefined") return; + window.dispatchEvent( + new CustomEvent(SOUNDCLAW_PLAYBACK_STARTED_EVENT, { + detail: { startedAt: Date.now() }, + }), + ); +}; + +type JukeboxView = "setup" | "player"; + +type JukeboxStore = { + // Auth state. + token: string | null; + clientId: string; + view: JukeboxView; + + // Player state. + playerState: PlayerState | null; + searchResults: SpotifyTrack[]; + searchQuery: string; + isSearching: boolean; + isLoadingPlayer: boolean; + error: string | null; + + // Actions. + init: () => void; + setClientId: (id: string) => void; + setToken: (token: string) => void; + disconnect: () => void; + refreshPlayer: () => Promise; + search: (query: string) => Promise; + setSearchQuery: (q: string) => void; + play: (uri: string) => Promise; + pause: () => Promise; + resume: () => Promise; + next: () => Promise; + previous: () => Promise; + volume: (percent: number) => Promise; +}; + +export const useJukeboxStore = create((set, get) => ({ + token: null, + clientId: "", + view: "setup", + playerState: null, + searchResults: [], + searchQuery: "", + isSearching: false, + isLoadingPlayer: false, + error: null, + + init: () => { + const token = loadToken(); + const clientId = loadClientId(); + set({ + token, + clientId, + view: token ? "player" : "setup", + }); + }, + + setClientId: (id) => { + saveClientId(id); + set({ clientId: id }); + }, + + setToken: (token) => { + set({ token, view: "player" }); + }, + + disconnect: () => { + clearToken(); + set({ token: null, view: "setup", playerState: null, searchResults: [], searchQuery: "" }); + }, + + refreshPlayer: async () => { + const { token } = get(); + if (!token) return; + set({ isLoadingPlayer: true, error: null }); + try { + const playerState = await fetchPlayerState(token); + set({ playerState, isLoadingPlayer: false }); + } catch { + set({ isLoadingPlayer: false, error: "Could not reach Spotify." }); + } + }, + + search: async (query) => { + const { token } = get(); + if (!token || !query.trim()) return; + set({ isSearching: true, error: null }); + try { + const results = await searchTracks(token, query); + set({ searchResults: results, isSearching: false }); + } catch { + set({ isSearching: false, error: "Search failed." }); + } + }, + + setSearchQuery: (q) => set({ searchQuery: q }), + + play: async (uri) => { + const { token, playerState } = get(); + if (!token) return; + set({ error: null }); + try { + await playTrack(token, uri, playerState?.deviceId); + // Brief delay for Spotify to update state. + await new Promise((r) => setTimeout(r, 500)); + await get().refreshPlayer(); + emitPlaybackStarted(); + } catch { + set({ error: "Playback failed. Make sure Spotify is open on a device." }); + } + }, + + pause: async () => { + const { token } = get(); + if (!token) return; + await pausePlayback(token); + set((s) => ({ + playerState: s.playerState ? { ...s.playerState, isPlaying: false } : null, + })); + }, + + resume: async () => { + const { token } = get(); + if (!token) return; + await resumePlayback(token); + set((s) => ({ + playerState: s.playerState ? { ...s.playerState, isPlaying: true } : null, + })); + emitPlaybackStarted(); + }, + + next: async () => { + const { token } = get(); + if (!token) return; + await skipToNext(token); + await new Promise((r) => setTimeout(r, 500)); + await get().refreshPlayer(); + }, + + previous: async () => { + const { token } = get(); + if (!token) return; + await skipToPrevious(token); + await new Promise((r) => setTimeout(r, 500)); + await get().refreshPlayer(); + }, + + volume: async (percent) => { + const { token } = get(); + if (!token) return; + await setVolume(token, percent); + set((s) => ({ + playerState: s.playerState ? { ...s.playerState, volumePercent: percent } : null, + })); + }, +})); + +export const SOUNDCLAW_PLAYBACK_STARTED_EVENT_NAME = SOUNDCLAW_PLAYBACK_STARTED_EVENT; diff --git a/src/lib/office/eventTriggers.ts b/src/lib/office/eventTriggers.ts index 76b16f9..db607d7 100644 --- a/src/lib/office/eventTriggers.ts +++ b/src/lib/office/eventTriggers.ts @@ -40,10 +40,7 @@ import { resolveOfficeStandupDirective, resolveOfficeTextDirective, } from "@/lib/office/deskDirectives"; -import { - extractText, - extractThinking, -} from "@/lib/text/message-extract"; +import { extractText, extractThinking } from "@/lib/text/message-extract"; import { randomUUID } from "@/lib/uuid"; // Office animation is derived in two passes: @@ -120,9 +117,11 @@ export type OfficeAnimationTriggerState = { export type OfficeAnimationState = { awaitingApprovalByAgentId: BooleanByAgentId; cleaningCues: OfficeCleaningCue[]; + danceUntilByAgentId: NumberByAgentId; deskHoldByAgentId: BooleanByAgentId; githubHoldByAgentId: BooleanByAgentId; gymHoldByAgentId: BooleanByAgentId; + jukeboxHoldByAgentId: BooleanByAgentId; manualGymUntilByAgentId: NumberByAgentId; pendingStandupRequest: OfficeStandupTriggerRequest | null; phoneBoothHoldByAgentId: BooleanByAgentId; @@ -136,14 +135,11 @@ export type OfficeAnimationState = { workingUntilByAgentId: NumberByAgentId; }; -const emptyObject = >(): T => ({} as T); +const emptyObject = >(): T => ({}) as T; const normalizeCommandText = (value: string | null | undefined): string => { if (!value) return ""; - return value - .trim() - .toLowerCase() - .replace(/\s+/g, " "); + return value.trim().toLowerCase().replace(/\s+/g, " "); }; const buildStableLatestRequestSeed = (value: string): string => { @@ -167,7 +163,8 @@ const pruneStringMap = ( ): StringByAgentId => Object.fromEntries( Object.entries(source).filter( - ([agentId, value]) => activeAgentIds.has(agentId) && value.trim().length > 0, + ([agentId, value]) => + activeAgentIds.has(agentId) && value.trim().length > 0, ), ); @@ -181,7 +178,8 @@ const prunePhoneCallMap = ( activeAgentIds.has(agentId) && Boolean(request?.callee?.trim()) && (request.phase === "needs_message" || - (request.phase === "ready_to_call" && Boolean(request.message?.trim()))), + (request.phase === "ready_to_call" && + Boolean(request.message?.trim()))), ), ); @@ -195,7 +193,8 @@ const pruneTextMessageMap = ( activeAgentIds.has(agentId) && Boolean(request?.recipient?.trim()) && (request.phase === "needs_message" || - (request.phase === "ready_to_send" && Boolean(request.message?.trim()))), + (request.phase === "ready_to_send" && + Boolean(request.message?.trim()))), ), ); @@ -220,7 +219,9 @@ const resolveMessageRole = (message: unknown): string | null => { return typeof role === "string" ? role : null; }; -const resolveChatPayloadRole = (payload: ChatEventPayload | undefined): string | null => { +const resolveChatPayloadRole = ( + payload: ChatEventPayload | undefined, +): string | null => { if (!payload) return null; const messageRole = resolveMessageRole(payload.message); if (messageRole) return messageRole; @@ -231,19 +232,20 @@ const resolveChatPayloadRole = (payload: ChatEventPayload | undefined): string | return typeof payloadRole === "string" ? payloadRole : null; }; -const isUserLikeChatRole = (role: string | null, state: ChatEventPayload["state"]): boolean => { +const isUserLikeChatRole = ( + role: string | null, + state: ChatEventPayload["state"], +): boolean => { if (role === "user" || role === "human" || role === "input") return true; if (role === "system") return state === "final"; return role === null && state === "final"; }; -const resolveLatestDirective = ( - params: { - lastUserMessage: string | null | undefined; - transcriptEntries: TranscriptEntry[] | undefined; - resolver: (value: string | null | undefined) => TDirective | null; - }, -): LatestDirective | null => { +const resolveLatestDirective = (params: { + lastUserMessage: string | null | undefined; + transcriptEntries: TranscriptEntry[] | undefined; + resolver: (value: string | null | undefined) => TDirective | null; +}): LatestDirective | null => { const latestMessageDirective = params.resolver(params.lastUserMessage); if (latestMessageDirective) { const text = params.lastUserMessage?.trim() ?? ""; @@ -253,10 +255,17 @@ const resolveLatestDirective = ( text, }; } - if (!Array.isArray(params.transcriptEntries) || params.transcriptEntries.length === 0) { + if ( + !Array.isArray(params.transcriptEntries) || + params.transcriptEntries.length === 0 + ) { return null; } - for (let index = params.transcriptEntries.length - 1; index >= 0; index -= 1) { + for ( + let index = params.transcriptEntries.length - 1; + index >= 0; + index -= 1 + ) { const entry = params.transcriptEntries[index]; if (!entry || entry.role !== "user") continue; const directive = params.resolver(entry.text); @@ -270,17 +279,23 @@ const resolveLatestDirective = ( return null; }; -const isTransientBoothRequestFresh = (requestedAt: number, nowMs: number): boolean => - nowMs - requestedAt <= TRANSIENT_BOOTH_RESTORE_MAX_AGE_MS; +const isTransientBoothRequestFresh = ( + requestedAt: number, + nowMs: number, +): boolean => nowMs - requestedAt <= TRANSIENT_BOOTH_RESTORE_MAX_AGE_MS; const maybeResolveCompletedPhoneCallRequest = ( current: OfficePhoneCallRequest | null, line: string, ): OfficePhoneCallRequest | null => { if (!current) return null; - const match = line.match(/^\[phone booth\]\s*Call with\s+(.+)\s+finished\.$/i); + const match = line.match( + /^\[phone booth\]\s*Call with\s+(.+)\s+finished\.$/i, + ); if (!match) return current; - return normalizeCommandText(match[1]) === normalizeCommandText(current.callee) ? null : current; + return normalizeCommandText(match[1]) === normalizeCommandText(current.callee) + ? null + : current; }; const maybeResolveCompletedTextMessageRequest = ( @@ -288,9 +303,12 @@ const maybeResolveCompletedTextMessageRequest = ( line: string, ): OfficeTextMessageRequest | null => { if (!current) return null; - const match = line.match(/^\[messaging booth\]\s*Message to\s+(.+)\s+sent\.$/i); + const match = line.match( + /^\[messaging booth\]\s*Message to\s+(.+)\s+sent\.$/i, + ); if (!match) return current; - return normalizeCommandText(match[1]) === normalizeCommandText(current.recipient) + return normalizeCommandText(match[1]) === + normalizeCommandText(current.recipient) ? null : current; }; @@ -361,7 +379,9 @@ const resolveLatestPhoneCallRequest = (params: { } } if (!current) return null; - return isTransientBoothRequestFresh(current.requestedAt, params.nowMs) ? current : null; + return isTransientBoothRequestFresh(current.requestedAt, params.nowMs) + ? current + : null; }; const resolveLatestTextMessageRequest = (params: { @@ -430,7 +450,9 @@ const resolveLatestTextMessageRequest = (params: { } } if (!current) return null; - return isTransientBoothRequestFresh(current.requestedAt, params.nowMs) ? current : null; + return isTransientBoothRequestFresh(current.requestedAt, params.nowMs) + ? current + : null; }; const resolveAgentIdForSessionKey = ( @@ -439,7 +461,9 @@ const resolveAgentIdForSessionKey = ( ): string | null => { const trimmed = sessionKey?.trim() ?? ""; if (!trimmed) return null; - const matched = agents.find((agent) => isSameSessionKey(agent.sessionKey, trimmed)); + const matched = agents.find((agent) => + isSameSessionKey(agent.sessionKey, trimmed), + ); if (matched) return matched.agentId; return parseAgentIdFromSessionKey(trimmed); }; @@ -501,13 +525,13 @@ const hasOtherOfficeDirective = ( ): boolean => Boolean( snapshot.desk || - snapshot.github || - snapshot.gym || - snapshot.qa || - snapshot.art || - snapshot.standup || - snapshot.call || - snapshot.text, + snapshot.github || + snapshot.gym || + snapshot.qa || + snapshot.art || + snapshot.standup || + snapshot.call || + snapshot.text, ); const resolvePhoneCallFollowUpRequest = (params: { @@ -522,11 +546,14 @@ const resolvePhoneCallFollowUpRequest = (params: { const message = params.message.trim(); if (!message) return null; return { - key: buildPhoneCallDirectiveKey({ - callee: params.current.callee, - phase: "ready_to_call", - message, - }, params.requestSeed ?? String(params.requestedAt)), + key: buildPhoneCallDirectiveKey( + { + callee: params.current.callee, + phase: "ready_to_call", + message, + }, + params.requestSeed ?? String(params.requestedAt), + ), callee: params.current.callee, message, phase: "ready_to_call", @@ -587,7 +614,10 @@ const pruneOfficeAnimationTriggerState = ( state.githubDirectiveKeyByAgentId, activeAgentIds, ), - githubHoldByAgentId: pruneBooleanMap(state.githubHoldByAgentId, activeAgentIds), + githubHoldByAgentId: pruneBooleanMap( + state.githubHoldByAgentId, + activeAgentIds, + ), gymCooldownUntilByAgentId: pruneFutureMap( state.gymCooldownUntilByAgentId, activeAgentIds, @@ -606,7 +636,10 @@ const pruneOfficeAnimationTriggerState = ( state.qaDirectiveKeyByAgentId, activeAgentIds, ), - phoneCallByAgentId: prunePhoneCallMap(state.phoneCallByAgentId, activeAgentIds), + phoneCallByAgentId: prunePhoneCallMap( + state.phoneCallByAgentId, + activeAgentIds, + ), phoneCallDirectiveKeyByAgentId: pruneStringMap( state.phoneCallDirectiveKeyByAgentId, activeAgentIds, @@ -686,7 +719,10 @@ const recordThinkingActivity = ( nowMs: number, ): NumberByAgentId => ({ ...current, - [agentId]: Math.max(current[agentId] ?? 0, nowMs + THINKING_ACTIVITY_LATCH_MS), + [agentId]: Math.max( + current[agentId] ?? 0, + nowMs + THINKING_ACTIVITY_LATCH_MS, + ), }); const applyUserMessageTriggers = (params: { @@ -717,7 +753,8 @@ const applyUserMessageTriggers = (params: { if (githubDirective) { const directiveKey = normalizeCommandText(params.message); const isSuppressed = - next.suppressedGithubDirectiveKeyByAgentId[params.agentId] === directiveKey; + next.suppressedGithubDirectiveKeyByAgentId[params.agentId] === + directiveKey; next = { ...next, githubDirectiveKeyByAgentId: { @@ -769,10 +806,7 @@ const applyUserMessageTriggers = (params: { }, }; } - if ( - params.agentId === "main" && - intentSnapshot.standup === "standup" - ) { + if (params.agentId === "main" && intentSnapshot.standup === "standup") { const requestKey = normalizeCommandText(params.message); if (next.pendingStandupRequest?.key !== requestKey) { next = { @@ -862,33 +896,34 @@ const applyUserMessageTriggers = (params: { return next; }; -export const createOfficeAnimationTriggerState = (): OfficeAnimationTriggerState => ({ - cleaningCues: [], - deskDirectiveKeyByAgentId: emptyObject(), - deskHoldByAgentId: emptyObject(), - githubDirectiveKeyByAgentId: emptyObject(), - githubHoldByAgentId: emptyObject(), - gymCooldownUntilByAgentId: emptyObject(), - lastManualGymCommandKeyByAgentId: emptyObject(), - manualGymUntilByAgentId: emptyObject(), - pendingStandupRequest: null, - phoneCallByAgentId: emptyObject(), - phoneCallDirectiveKeyByAgentId: emptyObject(), - qaDirectiveKeyByAgentId: emptyObject(), - qaHoldByAgentId: emptyObject(), - sessionEpochSnapshot: {}, - skillGymDirectiveKeyByAgentId: emptyObject(), - skillGymHoldByAgentId: emptyObject(), - streamingUntilByAgentId: emptyObject(), - suppressedPhoneCallDirectiveKeyByAgentId: emptyObject(), - suppressedGithubDirectiveKeyByAgentId: emptyObject(), - suppressedQaDirectiveKeyByAgentId: emptyObject(), - suppressedTextMessageDirectiveKeyByAgentId: emptyObject(), - textMessageByAgentId: emptyObject(), - textMessageDirectiveKeyByAgentId: emptyObject(), - thinkingUntilByAgentId: emptyObject(), - workingUntilByAgentId: emptyObject(), -}); +export const createOfficeAnimationTriggerState = + (): OfficeAnimationTriggerState => ({ + cleaningCues: [], + deskDirectiveKeyByAgentId: emptyObject(), + deskHoldByAgentId: emptyObject(), + githubDirectiveKeyByAgentId: emptyObject(), + githubHoldByAgentId: emptyObject(), + gymCooldownUntilByAgentId: emptyObject(), + lastManualGymCommandKeyByAgentId: emptyObject(), + manualGymUntilByAgentId: emptyObject(), + pendingStandupRequest: null, + phoneCallByAgentId: emptyObject(), + phoneCallDirectiveKeyByAgentId: emptyObject(), + qaDirectiveKeyByAgentId: emptyObject(), + qaHoldByAgentId: emptyObject(), + sessionEpochSnapshot: {}, + skillGymDirectiveKeyByAgentId: emptyObject(), + skillGymHoldByAgentId: emptyObject(), + streamingUntilByAgentId: emptyObject(), + suppressedPhoneCallDirectiveKeyByAgentId: emptyObject(), + suppressedGithubDirectiveKeyByAgentId: emptyObject(), + suppressedQaDirectiveKeyByAgentId: emptyObject(), + suppressedTextMessageDirectiveKeyByAgentId: emptyObject(), + textMessageByAgentId: emptyObject(), + textMessageDirectiveKeyByAgentId: emptyObject(), + thinkingUntilByAgentId: emptyObject(), + workingUntilByAgentId: emptyObject(), + }); export const reduceOfficeAnimationTriggerEvent = (params: { agents: AgentState[]; @@ -897,7 +932,11 @@ export const reduceOfficeAnimationTriggerEvent = (params: { state: OfficeAnimationTriggerState; }): OfficeAnimationTriggerState => { const nowMs = params.nowMs ?? Date.now(); - let next = pruneOfficeAnimationTriggerState(params.state, params.agents, nowMs); + let next = pruneOfficeAnimationTriggerState( + params.state, + params.agents, + nowMs, + ); const kind = classifyGatewayEventKind(params.event.event); if (kind === "runtime-chat") { @@ -908,7 +947,8 @@ export const reduceOfficeAnimationTriggerEvent = (params: { ); if (!payload || !agentId) return next; const messageText = extractText(payload.message)?.trim() ?? ""; - const thinkingText = extractThinking(payload.message ?? payload)?.trim() ?? ""; + const thinkingText = + extractThinking(payload.message ?? payload)?.trim() ?? ""; const role = resolveChatPayloadRole(payload); if (payload.runId) { next = { @@ -1015,7 +1055,9 @@ export const reduceOfficeAnimationTriggerEvent = (params: { const resolved = parseExecApprovalResolved(params.event); if (resolved) { - const approvalAgentId = params.agents.find((agent) => agent.awaitingUserInput)?.agentId; + const approvalAgentId = params.agents.find( + (agent) => agent.awaitingUserInput, + )?.agentId; if (approvalAgentId) { next = { ...next, @@ -1039,7 +1081,11 @@ export const reconcileOfficeAnimationTriggerState = (params: { // Reconciliation is the durable source of truth. It replays the latest user-visible intent // from current agent state so recovered history can restore holds even when chat events were missed. const nowMs = params.nowMs ?? Date.now(); - const next = pruneOfficeAnimationTriggerState(params.state, params.agents, nowMs); + const next = pruneOfficeAnimationTriggerState( + params.state, + params.agents, + nowMs, + ); const activeAgentIds = new Set(params.agents.map((agent) => agent.agentId)); const currentImmediateGymKeys = pruneStringMap( @@ -1100,7 +1146,8 @@ export const reconcileOfficeAnimationTriggerState = (params: { }); if (githubDirective) { githubDirectiveKeyByAgentId[agentId] = githubDirective.key; - const suppressedKey = next.suppressedGithubDirectiveKeyByAgentId[agentId] ?? ""; + const suppressedKey = + next.suppressedGithubDirectiveKeyByAgentId[agentId] ?? ""; if ( githubDirective.directive !== "release" && suppressedKey !== githubDirective.key @@ -1118,8 +1165,12 @@ export const reconcileOfficeAnimationTriggerState = (params: { }); if (qaDirective) { qaDirectiveKeyByAgentId[agentId] = qaDirective.key; - const suppressedKey = next.suppressedQaDirectiveKeyByAgentId[agentId] ?? ""; - if (qaDirective.directive !== "release" && suppressedKey !== qaDirective.key) { + const suppressedKey = + next.suppressedQaDirectiveKeyByAgentId[agentId] ?? ""; + if ( + qaDirective.directive !== "release" && + suppressedKey !== qaDirective.key + ) { qaHoldByAgentId[agentId] = true; } } else if (next.qaHoldByAgentId[agentId]) { @@ -1190,7 +1241,9 @@ export const reconcileOfficeAnimationTriggerState = (params: { previous: next.sessionEpochSnapshot, agents: params.agents, }); - const agentMap = new Map(params.agents.map((agent) => [agent.agentId, agent])); + const agentMap = new Map( + params.agents.map((agent) => [agent.agentId, agent]), + ); const cleaningCues = [...next.cleaningCues]; for (const agentId of triggeredAgentIds) { const agent = agentMap.get(agentId); @@ -1248,7 +1301,8 @@ export const clearOfficeAnimationTriggerHold = (params: { }; } if (params.hold === "call") { - const directiveKey = next.phoneCallDirectiveKeyByAgentId[params.agentId] ?? ""; + const directiveKey = + next.phoneCallDirectiveKeyByAgentId[params.agentId] ?? ""; const phoneCallByAgentId = { ...next.phoneCallByAgentId }; delete phoneCallByAgentId[params.agentId]; return { @@ -1263,7 +1317,8 @@ export const clearOfficeAnimationTriggerHold = (params: { }; } if (params.hold === "text") { - const directiveKey = next.textMessageDirectiveKeyByAgentId[params.agentId] ?? ""; + const directiveKey = + next.textMessageDirectiveKeyByAgentId[params.agentId] ?? ""; const textMessageByAgentId = { ...next.textMessageByAgentId }; delete textMessageByAgentId[params.agentId]; return { @@ -1305,6 +1360,7 @@ export const buildOfficeAnimationState = (params: { const awaitingApprovalByAgentId: BooleanByAgentId = {}; const deskHoldByAgentId: BooleanByAgentId = {}; const gymHoldByAgentId: BooleanByAgentId = {}; + const jukeboxHoldByAgentId: BooleanByAgentId = {}; const phoneBoothHoldByAgentId: BooleanByAgentId = {}; const phoneCallByAgentId: PhoneCallByAgentId = {}; const smsBoothHoldByAgentId: BooleanByAgentId = {}; @@ -1353,9 +1409,11 @@ export const buildOfficeAnimationState = (params: { return { awaitingApprovalByAgentId, cleaningCues: params.state.cleaningCues, + danceUntilByAgentId: {}, deskHoldByAgentId, githubHoldByAgentId: params.state.githubHoldByAgentId, gymHoldByAgentId, + jukeboxHoldByAgentId, manualGymUntilByAgentId: params.state.manualGymUntilByAgentId, pendingStandupRequest: params.state.pendingStandupRequest, phoneBoothHoldByAgentId, diff --git a/src/lib/office/places.ts b/src/lib/office/places.ts index e11023d..897f64c 100644 --- a/src/lib/office/places.ts +++ b/src/lib/office/places.ts @@ -3,17 +3,20 @@ export const OFFICE_INTERACTION_TARGETS = [ "server_room", "meeting_room", "gym", + "jukebox", "qa_lab", "sms_booth", "phone_booth", ] as const; -export type OfficeInteractionTargetId = (typeof OFFICE_INTERACTION_TARGETS)[number]; +export type OfficeInteractionTargetId = + (typeof OFFICE_INTERACTION_TARGETS)[number]; export const OFFICE_SKILL_TRIGGER_MOVEMENT_TARGETS = [ "desk", "github", "gym", + "jukebox", "qa_lab", ] as const; @@ -24,6 +27,7 @@ type OfficeSkillTriggerAnimationHoldKey = | "deskHoldByAgentId" | "githubHoldByAgentId" | "gymHoldByAgentId" + | "jukeboxHoldByAgentId" | "qaHoldByAgentId"; export const OFFICE_SKILL_TRIGGER_PLACE_REGISTRY: Record< @@ -51,6 +55,11 @@ export const OFFICE_SKILL_TRIGGER_PLACE_REGISTRY: Record< animationHoldKey: "gymHoldByAgentId", alsoSetsSkillGymHold: true, }, + jukebox: { + label: "Jukebox", + interactionTarget: "jukebox", + animationHoldKey: "jukeboxHoldByAgentId", + }, qa_lab: { label: "QA Lab", interactionTarget: "qa_lab", @@ -85,6 +94,20 @@ export const DEFAULT_SKILL_TRIGGER_FALLBACKS_BY_SKILL_KEY: Record< movementTarget: "desk", skipIfAlreadyThere: true, }, + soundclaw: { + anyPhrases: [ + "spotify", + "play a song", + "play this song", + "play music", + "play a playlist", + "find a song", + "queue this song", + "music link", + ], + movementTarget: "jukebox", + skipIfAlreadyThere: true, + }, }; export const buildOfficeSkillTriggerHoldMaps = ( @@ -93,6 +116,7 @@ export const buildOfficeSkillTriggerHoldMaps = ( deskHoldByAgentId: Record; githubHoldByAgentId: Record; gymHoldByAgentId: Record; + jukeboxHoldByAgentId: Record; qaHoldByAgentId: Record; skillGymHoldByAgentId: Record; } => { @@ -100,6 +124,7 @@ export const buildOfficeSkillTriggerHoldMaps = ( deskHoldByAgentId: {} as Record, githubHoldByAgentId: {} as Record, gymHoldByAgentId: {} as Record, + jukeboxHoldByAgentId: {} as Record, qaHoldByAgentId: {} as Record, skillGymHoldByAgentId: {} as Record, }; diff --git a/src/lib/skills/catalog.ts b/src/lib/skills/catalog.ts index a469956..1768761 100644 --- a/src/lib/skills/catalog.ts +++ b/src/lib/skills/catalog.ts @@ -1,6 +1,9 @@ -import type { RemovableSkillSource, SkillStatusEntry } from "@/lib/skills/types"; +import type { + RemovableSkillSource, + SkillStatusEntry, +} from "@/lib/skills/types"; -export type PackagedSkillId = "todo-board"; +export type PackagedSkillId = "soundclaw" | "todo-board"; export type PackagedSkillDefinition = { packageId: PackagedSkillId; @@ -30,20 +33,35 @@ const PACKAGED_SKILLS: PackagedSkillDefinition[] = [ creatorName: "iamlukethedev", creatorUrl: "http://x.com/iamlukethedev/", }, + { + packageId: "soundclaw", + skillKey: "soundclaw", + name: "soundclaw", + description: "Control Spotify playback, search music, and return shareable music links.", + installSource: "openclaw-workspace", + creatorName: "iamlukethedev", + creatorUrl: "https://github.com/iamlukethedev", + }, ]; -export const listPackagedSkills = (): PackagedSkillDefinition[] => [...PACKAGED_SKILLS]; +export const listPackagedSkills = (): PackagedSkillDefinition[] => [ + ...PACKAGED_SKILLS, +]; -export const getPackagedSkillById = (packageId: string): PackagedSkillDefinition | null => +export const getPackagedSkillById = ( + packageId: string, +): PackagedSkillDefinition | null => PACKAGED_SKILLS.find((skill) => skill.packageId === packageId) ?? null; -export const getPackagedSkillBySkillKey = (skillKey: string): PackagedSkillDefinition | null => { +export const getPackagedSkillBySkillKey = ( + skillKey: string, +): PackagedSkillDefinition | null => { const normalized = skillKey.trim(); return PACKAGED_SKILLS.find((skill) => skill.skillKey === normalized) ?? null; }; export const buildPackagedSkillStatusEntry = ( - skill: PackagedSkillDefinition + skill: PackagedSkillDefinition, ): SkillStatusEntry => ({ name: skill.name, description: skill.description, @@ -62,11 +80,13 @@ export const buildPackagedSkillStatusEntry = ( install: [], }); -export const appendPackagedSkillsToMarketplace = (skills: SkillStatusEntry[]): SkillStatusEntry[] => { +export const appendPackagedSkillsToMarketplace = ( + skills: SkillStatusEntry[], +): SkillStatusEntry[] => { const presentKeys = new Set(skills.map((skill) => skill.skillKey.trim())); - const additions = PACKAGED_SKILLS.filter((skill) => !presentKeys.has(skill.skillKey)).map( - buildPackagedSkillStatusEntry - ); + const additions = PACKAGED_SKILLS.filter( + (skill) => !presentKeys.has(skill.skillKey), + ).map(buildPackagedSkillStatusEntry); if (additions.length === 0) { return skills; } diff --git a/src/lib/skills/marketplace.ts b/src/lib/skills/marketplace.ts index 461ee09..f474e09 100644 --- a/src/lib/skills/marketplace.ts +++ b/src/lib/skills/marketplace.ts @@ -42,11 +42,18 @@ export type SkillMarketplaceEntry = { missingDetails: string[]; }; -const SKILL_MARKETPLACE_OVERRIDES: Record> = { +const SKILL_MARKETPLACE_OVERRIDES: Record< + string, + Partial +> = { github: { category: "Engineering", tagline: "Turns repository operations into a one-step teammate workflow.", - capabilities: ["Pull request support", "Issue context", "Repository operations"], + capabilities: [ + "Pull request support", + "Issue context", + "Repository operations", + ], featured: true, editorBadge: "Popular", rating: 4.9, @@ -64,14 +71,19 @@ const SKILL_MARKETPLACE_OVERRIDES: Record { @@ -122,7 +148,9 @@ const buildFallbackCapabilities = (skill: SkillStatusEntry): string[] => { return capabilities.slice(0, 3); }; -const buildFallbackMetadata = (skill: SkillStatusEntry): SkillMarketplaceMetadata => { +const buildFallbackMetadata = ( + skill: SkillStatusEntry, +): SkillMarketplaceMetadata => { const normalizedKey = skill.skillKey.trim().toLowerCase(); const source = skill.source.trim(); const seed = hashString(`${normalizedKey}:${source}`); @@ -146,7 +174,9 @@ const buildFallbackMetadata = (skill: SkillStatusEntry): SkillMarketplaceMetadat : "Community"; return { category, - tagline: skill.description.trim() || `${titleCaseWords(skill.name)} capability pack.`, + tagline: + skill.description.trim() || + `${titleCaseWords(skill.name)} capability pack.`, trustLabel, capabilities: buildFallbackCapabilities(skill), featured: skill.bundled || source === "openclaw-managed", @@ -155,7 +185,9 @@ const buildFallbackMetadata = (skill: SkillStatusEntry): SkillMarketplaceMetadat }; }; -export const resolveSkillMarketplaceMetadata = (skill: SkillStatusEntry): SkillMarketplaceMetadata => { +export const resolveSkillMarketplaceMetadata = ( + skill: SkillStatusEntry, +): SkillMarketplaceMetadata => { const normalizedKey = skill.skillKey.trim().toLowerCase(); const fallback = buildFallbackMetadata(skill); const override = SKILL_MARKETPLACE_OVERRIDES[normalizedKey]; @@ -178,11 +210,15 @@ export const resolveSkillMarketplaceMetadata = (skill: SkillStatusEntry): SkillM }; }; -export const buildSkillMarketplaceEntry = (skill: SkillStatusEntry): SkillMarketplaceEntry => { +export const buildSkillMarketplaceEntry = ( + skill: SkillStatusEntry, +): SkillMarketplaceEntry => { const packagedSkill = getPackagedSkillBySkillKey(skill.skillKey); const missingDetails = buildSkillMissingDetails(skill); if (packagedSkill && !skill.baseDir.trim()) { - missingDetails.unshift("Install this packaged Claw3D skill to make it available on the gateway."); + missingDetails.unshift( + "Install this packaged Claw3D skill to make it available on the gateway.", + ); } return { skill, @@ -195,7 +231,7 @@ export const buildSkillMarketplaceEntry = (skill: SkillStatusEntry): SkillMarket }; export const buildSkillMarketplaceCollections = ( - skills: SkillStatusEntry[] + skills: SkillStatusEntry[], ): Array<{ id: SkillMarketplaceCollectionId; label: string; @@ -209,24 +245,40 @@ export const buildSkillMarketplaceCollections = ( entries: SkillMarketplaceEntry[]; }> = []; - const featured = entries.filter((entry) => entry.metadata.featured).slice(0, 6); + const featured = entries + .filter((entry) => entry.metadata.featured) + .slice(0, 6); if (featured.length > 0) { collections.push({ id: "featured", label: "Featured", entries: featured }); } - const claw3d = entries.filter((entry) => getPackagedSkillBySkillKey(entry.skill.skillKey)); + const claw3d = entries.filter((entry) => + getPackagedSkillBySkillKey(entry.skill.skillKey), + ); if (claw3d.length > 0) { collections.push({ id: "claw3d", label: "Claw3D", entries: claw3d }); } - const installed = entries.filter((entry) => entry.readiness === "ready" || entry.skill.disabled); + const installed = entries.filter( + (entry) => entry.readiness === "ready" || entry.skill.disabled, + ); if (installed.length > 0) { - collections.push({ id: "installed", label: "Installed", entries: installed }); + collections.push({ + id: "installed", + label: "Installed", + entries: installed, + }); } - const setupRequired = entries.filter((entry) => entry.readiness === "needs-setup"); + const setupRequired = entries.filter( + (entry) => entry.readiness === "needs-setup", + ); if (setupRequired.length > 0) { - collections.push({ id: "setup-required", label: "Needs setup", entries: setupRequired }); + collections.push({ + id: "setup-required", + label: "Needs setup", + entries: setupRequired, + }); } for (const group of sourceGroups) { diff --git a/src/lib/skills/packaged.ts b/src/lib/skills/packaged.ts index 6d1f8f2..89043e9 100644 --- a/src/lib/skills/packaged.ts +++ b/src/lib/skills/packaged.ts @@ -140,6 +140,53 @@ const TODO_BOARD_EXAMPLE_JSON = `{ } `; +// Keep this string synchronized with assets/skills/soundclaw/SKILL.md. +const SOUNDCLAW_SKILL_MD = `--- +name: soundclaw +description: Control Spotify playback, search music, and return shareable music links. +metadata: {"openclaw":{"skillKey":"soundclaw"}} +--- + +# SOUNDCLAW + +Use this skill when the user wants an agent to search for music, play a song or playlist, control Spotify playback, or send back a shareable Spotify link on the same channel the request came from. + +## Trigger + +\`\`\`json +{ + "activation": { + "anyPhrases": [ + "spotify", + "play a song", + "play this song", + "play music", + "play a playlist", + "find a song", + "queue this song", + "music link" + ] + }, + "movement": { + "target": "jukebox", + "skipIfAlreadyThere": true + } +} +\`\`\` + +When this skill is activated, the agent should walk to the office jukebox before handling the request. + +- Treat requests from Telegram or any other external surface as valid triggers when they ask for Spotify playback, search, queueing, or music-link sharing. +- The physical behavior for this skill is: go to the jukebox, perform the music-selection workflow, then report the result. +- If the agent is already at the jukebox, continue without adding extra movement narration. + +## Channel behavior + +- Reply on the same active channel or session that received the request. +- If playback cannot start but a matching track, album, or playlist is found, send back the best Spotify link instead of failing silently. +- If multiple matches are plausible, ask a clarifying question instead of guessing. +`; + const PACKAGED_SKILL_FILES: Record = { "todo-board": [ { @@ -151,9 +198,17 @@ const PACKAGED_SKILL_FILES: Record = { content: TODO_BOARD_EXAMPLE_JSON, }, ], + soundclaw: [ + { + relativePath: "SKILL.md", + content: SOUNDCLAW_SKILL_MD, + }, + ], }; -export const readPackagedSkillFiles = (packageId: string): PackagedSkillFile[] => { +export const readPackagedSkillFiles = ( + packageId: string, +): PackagedSkillFile[] => { const files = PACKAGED_SKILL_FILES[packageId]; if (!files || files.length === 0) { throw new Error(`Packaged skill assets are missing: ${packageId}`); diff --git a/tests/unit/skillTriggers.test.ts b/tests/unit/skillTriggers.test.ts index 9e7a5a4..79b029f 100644 --- a/tests/unit/skillTriggers.test.ts +++ b/tests/unit/skillTriggers.test.ts @@ -54,9 +54,16 @@ describe("skill triggers", () => { }); it("keeps trigger places and fallback definitions in one central registry", () => { - expect(OFFICE_SKILL_TRIGGER_PLACE_REGISTRY.desk.interactionTarget).toBe("desk"); - expect(OFFICE_SKILL_TRIGGER_PLACE_REGISTRY.github.interactionTarget).toBe("server_room"); - expect(DEFAULT_SKILL_TRIGGER_FALLBACKS_BY_SKILL_KEY["todo-board"]?.movementTarget).toBe("desk"); + expect(OFFICE_SKILL_TRIGGER_PLACE_REGISTRY.desk.interactionTarget).toBe( + "desk", + ); + expect(OFFICE_SKILL_TRIGGER_PLACE_REGISTRY.github.interactionTarget).toBe( + "server_room", + ); + expect( + DEFAULT_SKILL_TRIGGER_FALLBACKS_BY_SKILL_KEY["todo-board"] + ?.movementTarget, + ).toBe("desk"); }); it("builds animation hold maps from the central place registry", () => {