Add tool-call disambiguation test harness for search_spotify/control_playback

Renders the Extended OpenAI Conversation prompt/function definitions and
checks the model's first tool-call choice against test phrases, bypassing
Home Assistant.

current-functions.txt

@@ -0,0 +1,225 @@
+- spec:
+    name: execute_services
+    description: Use this function to execute service of devices in Home Assistant.
+    parameters:
+      type: object
+      properties:
+        list:
+          type: array
+          items:
+            type: object
+            properties:
+              domain:
+                type: string
+                description: The domain of the service
+              service:
+                type: string
+                description: The service to be called
+              service_data:
+                type: object
+                description: The service data object to indicate what to control.
+                properties:
+                  entity_id:
+                    type: string
+                    description: The entity_id retrieved from available devices. It
+                      must start with domain, followed by dot character.
+                required:
+                - entity_id
+            required:
+            - domain
+            - service
+            - service_data
+  function:
+    type: native
+    name: execute_service
+
+
+# Spotify Music Control Functions
+
+- spec:
+    name: search_spotify
+    description: >-
+      Search Spotify for an artist, album, track, or playlist and return the Spotify URI.
+      Use this before playing music to get the URI. Use this whenever the user asks to play
+      or queue a specific song, artist, album, or playlist — even if the name contains a
+      number (e.g. "Back on 74", "1979", "Blink 182"). A number that is part of a
+      song/artist/album name is NOT a volume level; only use control_playback's volume_set
+      if the user explicitly says "volume" or "percent".
+    parameters:
+      type: object
+      properties:
+        query:
+          type: string
+          description: Artist, album, track, or playlist name to search for (e.g., "Coldplay", "Parachutes", "Yellow", "Today's Top Hits")
+        type:
+          type: string
+          enum: [artist, album, track, playlist]
+          description: Type of content to search for. Playlist searches check your personal playlists first, then fall back to public Spotify playlists.
+      required:
+        - query
+        - type
+  function:
+    type: script
+    sequence:
+      - service: spotify_voice_assistant.search
+        response_variable: _function_result
+        data:
+          query: "{{ query }}"
+          type: "{{ type }}"
+
+- spec:
+    name: play_music
+    description: Play music immediately on a Spotify Connect device using a Spotify URI, replacing current playback. You must first call search_spotify to get the URI. Use this when user says "play".
+    parameters:
+      type: object
+      properties:
+        spotify_uri:
+          type: string
+          description: Spotify URI from search_spotify (e.g., "spotify:artist:4gzpq5DPGxSnKTe4SA8HAU")
+        type:
+          type: string
+          enum: [artist, album, track, playlist]
+          description: The exact "type" value returned by search_spotify for this URI. Required — passing the wrong value causes playback to silently fail.
+        room:
+          type: string
+          description: Spotify Connect source/device name to play on, matching an entry in media_player.spotify_bhetherman's source_list exactly (e.g., "Living Room")
+      required:
+        - spotify_uri
+        - type
+        - room
+  function:
+    type: script
+    sequence:
+      - service: media_player.select_source
+        target:
+          entity_id: media_player.spotify_bhetherman
+        data:
+          source: "{{ room }}"
+      - delay: "00:00:02"
+      - service: media_player.play_media
+        target:
+          entity_id: media_player.spotify_bhetherman
+        data:
+          media_content_id: "{{ spotify_uri }}"
+          media_content_type: "{{ type }}"
+          enqueue: replace
+
+- spec:
+    name: queue_music
+    description: >-
+      Queue or play music using a Spotify URI from search_spotify. Use this whenever the
+      user says "queue" or "add to queue" — always use this function for queue requests,
+      regardless of whether the content is a track, album, artist, or playlist; the
+      correct behavior for each type is handled automatically. (Individual tracks are
+      added to the queue without interrupting playback; since Spotify cannot queue a whole
+      album/artist/playlist, those start playing immediately instead.)
+    parameters:
+      type: object
+      properties:
+        spotify_uri:
+          type: string
+          description: Spotify URI from search_spotify (e.g., "spotify:track:4gzpq5DPGxSnKTe4SA8HAU")
+        type:
+          type: string
+          enum: [artist, album, track, playlist]
+          description: The exact "type" value returned by search_spotify for this URI.
+        room:
+          type: string
+          description: Spotify Connect source/device name to queue on, matching an entry in media_player.spotify_bhetherman's source_list exactly (e.g., "Living Room")
+      required:
+        - spotify_uri
+        - type
+        - room
+  function:
+    type: script
+    sequence:
+      - service: media_player.select_source
+        target:
+          entity_id: media_player.spotify_bhetherman
+        data:
+          source: "{{ room }}"
+      - delay: "00:00:02"
+      - service: media_player.play_media
+        target:
+          entity_id: media_player.spotify_bhetherman
+        data:
+          media_content_id: "{{ spotify_uri }}"
+          media_content_type: "{{ type }}"
+          enqueue: "{{ 'add' if type == 'track' else 'replace' }}"
+
+- spec:
+    name: control_playback
+    description: >-
+      Control playback of music that is ALREADY PLAYING (pause, resume, stop, next track,
+      previous track, set volume, shuffle) on the currently active Spotify Connect session.
+      Only use volume_set when the user explicitly says the word "volume" or "percent"
+      (e.g. "turn the volume up", "set volume to 50", "50 percent volume").
+      Do NOT use this function to play a requested song, artist, or album — a number in a
+      song/artist/album name (e.g. "Back on 74", "Blink 182", "Sunset 2005") is part of the
+      title, never a volume level. Use search_spotify + play_music for any request to play
+      or start specific music, even if it contains a number.
+    parameters:
+      type: object
+      properties:
+        action:
+          type: string
+          enum: [pause, play, stop, next_track, previous_track, volume_set, shuffle_on, shuffle_off]
+          description: Playback control action
+        volume_level:
+          type: number
+          description: Volume level (0-100) for volume_set action only. Only set this when the user said the word "volume" or "percent".
+      required:
+        - action
+  function:
+    type: script
+    sequence:
+      - choose:
+          - conditions: "{{ action == 'pause' }}"
+            sequence:
+              - service: media_player.media_pause
+                target:
+                  entity_id: media_player.spotify_bhetherman
+          - conditions: "{{ action == 'play' }}"
+            sequence:
+              - service: media_player.media_play
+                target:
+                  entity_id: media_player.spotify_bhetherman
+          - conditions: "{{ action == 'stop' }}"
+            sequence:
+              # media_player.spotify_bhetherman does not support the media_stop
+              # service (STOP is not in its supported_features); Spotify Connect has
+              # no separate stop concept, so route "stop" through pause instead.
+              - service: media_player.media_pause
+                target:
+                  entity_id: media_player.spotify_bhetherman
+          - conditions: "{{ action == 'next_track' }}"
+            sequence:
+              - service: media_player.media_next_track
+                target:
+                  entity_id: media_player.spotify_bhetherman
+          - conditions: "{{ action == 'previous_track' }}"
+            sequence:
+              - service: media_player.media_previous_track
+                target:
+                  entity_id: media_player.spotify_bhetherman
+          - conditions: "{{ action == 'volume_set' }}"
+            sequence:
+              - service: media_player.volume_set
+                target:
+                  entity_id: media_player.spotify_bhetherman
+                data:
+                  volume_level: "{{ volume_level / 100 }}"
+          - conditions: "{{ action == 'shuffle_on' }}"
+            sequence:
+              - service: media_player.shuffle_set
+                target:
+                  entity_id: media_player.spotify_bhetherman
+                data:
+                  shuffle: true
+          - conditions: "{{ action == 'shuffle_off' }}"
+            sequence:
+              - service: media_player.shuffle_set
+                target:
+                  entity_id: media_player.spotify_bhetherman
+                data:
+                  shuffle: false

current-prompt.txt

@@ -0,0 +1,62 @@
+I want you to act as smart home manager of Home Assistant.
+I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language.
+
+Current Time: {{now()}}
+
+Available Devices:
+```csv
+entity_id,name,state,aliases
+{% for entity in exposed_entities -%}
+{{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{entity.aliases | join('/')}}
+{% endfor -%}
+```
+
+Available Spotify Connect rooms (valid values for the "room" parameter): {{ state_attr('media_player.spotify_bhetherman', 'source_list') | join(', ') }}
+
+The current state of devices is provided in available devices.
+Use execute_services function only for requested action, not for current states.
+Do not execute service without user's confirmation. Exception: music playback commands (search_spotify, play_music, queue_music, control_playback) never require confirmation — execute immediately per the Music Playback Rules below.
+Do not restate or appreciate what user says, rather make a quick inquiry.
+
+Music Playback Rules:
+
+CRITICAL: a number inside a song/artist/album name (e.g. "Back on 74", "1979", "Blink 182") is part of the title, NOT a volume level. Only treat a number as a volume level when the user says the word "volume" or "percent". When in doubt between search_spotify and control_playback's volume_set, prefer search_spotify.
+
+For any request to play or queue music, always call TWO functions in order, with no confirmation:
+1. search_spotify — to get the Spotify URI and its type
+2. play_music (if the user said "play") or queue_music (if the user said "queue"/"add to queue")
+
+When calling play_music or queue_music, always set type to the exact "type" value that search_spotify's result returned — never guess or hardcode it. Passing the wrong type silently breaks playback.
+
+RULE: if the request has the form "[title] by [artist]" (a specific song/album title followed by "by" and an artist name — no matter what words come before it, e.g. "play", "queue", "throw on", "put on", "the album", "the song"), the query is ONLY the title — always strip " by [artist]" from the end. Never include the word "by" or the artist name in the query for this pattern.
+- "Play Yellow by Coldplay" → query="Yellow"
+- "Throw on the album Parachutes by Coldplay" → query="Parachutes"
+- "Queue Wish You Were Here by Pink Floyd" → query="Wish You Were Here"
+
+How to fill in search_spotify(query, type) — match these examples:
+- "Play Coldplay" → query="Coldplay", type="artist"
+- "Play songs by Taylor Swift on the kitchen speaker" → query="Taylor Swift", type="artist"
+- "Play some music by Jungle in the living room" → query="Jungle", type="artist"
+- "Play back on 74 in the living room" → query="Back on 74", type="track" (this is a song title, not a volume command)
+- "Play the song Yellow" → query="Yellow", type="track"
+- "Play Yellow by Coldplay" → query="Yellow", type="track"
+- "Play 1979 by the Smashing Pumpkins" → query="1979", type="track" (a bare number here is a song title, not an album — default to "track" unless the user explicitly says "album")
+- "Queue Dark Side of the Moon in the kitchen" → query="Dark Side of the Moon", type="album"
+- "Add Wish You Were Here to queue in the bathroom" → query="Wish You Were Here", type="track"
+- "Queue Fred again in the bedroom" → query="Fred again", type="artist" (an artist name after "queue" with no specific song/album title named is still type="artist", same as "play")
+- "Play my workout playlist" → query="workout", type="playlist"
+- "Play jazz" → query="jazz", type="playlist"
+
+Volume/playback control examples (use control_playback, NOT search_spotify):
+- "Set the volume to 50" / "50 percent volume" → action="volume_set", volume_level=50
+- "Turn the volume up" → action="volume_set"
+- "Pause the music" → action="pause"
+- "Skip this song" → action="next_track"
+
+Room:
+- The "room" parameter for play_music/queue_music must exactly match one of the Available Spotify Connect rooms listed above.
+- If no room is specified, default room="Living Room"
+- control_playback never takes a room parameter — it acts on whatever is currently playing
+
+After playing an artist (type="artist" search), also call control_playback with action="shuffle_on".
+Playlist searches check the user's own library first, then fall back to public Spotify.

test_prompt.py

@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+"""Test search_spotify/control_playback tool-call disambiguation directly against
+the Ollama endpoint, bypassing Home Assistant entirely.
+
+Renders current-prompt.txt (with mock HA template values) and current-functions.txt
+(as OpenAI-style tool definitions) exactly as Extended OpenAI Conversation would send
+them, then checks whether the model picks the right tool + arguments for each test
+phrase. Only evaluates the FIRST tool call the model returns per test (i.e. whether
+it correctly picks search_spotify vs control_playback and fills in query/type/action) —
+it does not simulate the full multi-turn tool-result loop.
+"""
+import json
+import os
+import sys
+from datetime import datetime
+
+import requests
+import yaml
+from jinja2 import Template
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+
+OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://192.168.50.214:11434/v1")
+OLLAMA_API_KEY = os.environ.get("OLLAMA_API_KEY", "38ecbb5e71adf1306291e810826d965afe51fa27ff42e60f")
+MODEL = os.environ.get("OLLAMA_MODEL")  # set this to the exact `ollama list` tag
+
+MOCK_SOURCE_LIST = ["Living Room", "Web Player (Firefox)", "Bedroom"]
+
+
+def render_prompt():
+    with open(os.path.join(HERE, "current-prompt.txt")) as f:
+        raw = f.read()
+
+    def state_attr(entity_id, attr):
+        if entity_id == "media_player.spotify_bhetherman" and attr == "source_list":
+            return MOCK_SOURCE_LIST
+        return None
+
+    template = Template(raw)
+    return template.render(
+        now=lambda: datetime.now(),
+        exposed_entities=[],  # not relevant to these tests; kept empty on purpose
+        state_attr=state_attr,
+    )
+
+
+def load_tools():
+    with open(os.path.join(HERE, "current-functions.txt")) as f:
+        functions = yaml.safe_load(f)
+    tools = []
+    for entry in functions:
+        spec = entry["spec"]
+        tools.append({
+            "type": "function",
+            "function": {
+                "name": spec["name"],
+                "description": spec["description"],
+                "parameters": spec["parameters"],
+            },
+        })
+    return tools
+
+
+def call_model(system_prompt, user_message, tools):
+    if not MODEL:
+        sys.exit("Set OLLAMA_MODEL env var to the exact `ollama list` tag before running.")
+
+    resp = requests.post(
+        f"{OLLAMA_BASE_URL}/chat/completions",
+        headers={"Authorization": f"Bearer {OLLAMA_API_KEY}"},
+        json={
+            "model": MODEL,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_message},
+            ],
+            "tools": tools,
+            "tool_choice": "auto",
+        },
+        timeout=120,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def first_tool_call(response):
+    message = response["choices"][0]["message"]
+    tool_calls = message.get("tool_calls") or []
+    if not tool_calls:
+        return None, None, message.get("content")
+    call = tool_calls[0]
+    name = call["function"]["name"]
+    try:
+        args = json.loads(call["function"]["arguments"])
+    except (json.JSONDecodeError, TypeError):
+        args = call["function"]["arguments"]
+    return name, args, None
+
+
+REPEAT = int(os.environ.get("TEST_REPEAT", "15"))
+
+
+def _check_once(user_message, expect_name, expect_fn, system_prompt, tools):
+    response = call_model(system_prompt, user_message, tools)
+    name, args, content = first_tool_call(response)
+
+    if name is None:
+        return False, f"NO TOOL CALL — model replied with text instead: {content!r}"
+    if name != expect_name:
+        return False, f"WRONG FUNCTION — expected {expect_name!r}, got {name!r} ({args})"
+
+    ok, reason = expect_fn(args)
+    if ok:
+        return True, f"{name}({args})"
+    return False, f"{name}({args}) — {reason}"
+
+
+def _summarize(label, user_message, passed, attempts):
+    n = len(attempts)
+    print(f"\n=== {label} ===")
+    print(f"  user: {user_message!r}")
+    for i, (ok, detail) in enumerate(attempts, 1):
+        mark = "✅" if ok else "❌"
+        print(f"  [{i}/{n}] {mark} {detail}")
+
+    if passed == n:
+        print(f"  RESULT: {passed}/{n} — reliable pass")
+    elif passed == 0:
+        print(f"  RESULT: {passed}/{n} — reliable FAIL")
+    else:
+        print(f"  RESULT: {passed}/{n} — FLAKY (inconsistent across runs)")
+    return passed, n
+
+
+def check(label, user_message, expect_name, expect_fn, system_prompt, tools):
+    attempts = [_check_once(user_message, expect_name, expect_fn, system_prompt, tools) for _ in range(REPEAT)]
+    passed = sum(1 for ok, _ in attempts if ok)
+    return _summarize(label, user_message, passed, attempts)
+
+
+def eq_ci(a, b):
+    return isinstance(a, str) and a.strip().lower() == b.strip().lower()
+
+
+def contains_ci(a, b):
+    return isinstance(a, str) and b.strip().lower() in a.strip().lower()
+
+
+TESTS = [
+    (
+        "artist-only, generic 'music by X' phrasing",
+        "play music by Jungle in the living room",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Jungle") and eq_ci(args.get("type"), "artist")
+        else (False, f"expected query='Jungle' type='artist', got {args}"),
+    ),
+    (
+        "song title containing a number (previously misread as volume)",
+        "play back on 74 in the living room",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Back on 74") and eq_ci(args.get("type"), "track")
+        else (False, f"expected query='Back on 74' type='track', got {args}"),
+    ),
+    (
+        "specific title + artist (regression check — previously worked)",
+        "play Yellow by Coldplay",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Yellow") and eq_ci(args.get("type"), "track")
+        else (False, f"expected query='Yellow' type='track', got {args}"),
+    ),
+    (
+        "explicit volume command (must NOT be treated as a search)",
+        "set the volume to 50",
+        "control_playback",
+        lambda args: (True, None) if args.get("action") == "volume_set" and float(args.get("volume_level", -1)) == 50
+        else (False, f"expected action='volume_set' volume_level=50, got {args}"),
+    ),
+    (
+        "album title, different room",
+        "queue Dark Side of the Moon in the bedroom",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Dark Side of the Moon") and args.get("type") in ("album", "track")
+        else (False, f"expected query='Dark Side of the Moon' type in (album,track), got {args}"),
+    ),
+    (
+        "genre/mood, no artist",
+        "play some jazz",
+        "search_spotify",
+        lambda args: (True, None) if contains_ci(args.get("query"), "jazz") and eq_ci(args.get("type"), "playlist")
+        else (False, f"expected query~'jazz' type='playlist', got {args}"),
+    ),
+    (
+        "named playlist phrase",
+        "play my daily mix",
+        "search_spotify",
+        lambda args: (True, None) if contains_ci(args.get("query"), "daily mix") and eq_ci(args.get("type"), "playlist")
+        else (False, f"expected query~'daily mix' type='playlist', got {args}"),
+    ),
+    (
+        "number-titled track + explicit artist (by-stripping regression variant)",
+        "play 1979 by the Smashing Pumpkins",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "1979") and eq_ci(args.get("type"), "track")
+        else (False, f"expected query='1979' type='track', got {args}"),
+    ),
+    (
+        "number-titled track, no artist given",
+        "play 99 Luftballons",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "99 Luftballons") and eq_ci(args.get("type"), "track")
+        else (False, f"expected query='99 Luftballons' type='track', got {args}"),
+    ),
+    (
+        "artist name that itself contains a number",
+        "queue some Blink 182",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Blink 182") and eq_ci(args.get("type"), "artist")
+        else (False, f"expected query='Blink 182' type='artist', got {args}"),
+    ),
+    (
+        "colloquial phrasing ('put on')",
+        "put on some Fred again",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Fred again") and eq_ci(args.get("type"), "artist")
+        else (False, f"expected query='Fred again' type='artist', got {args}"),
+    ),
+    (
+        "explicit 'album' keyword + by-stripping",
+        "throw on the album Parachutes by Coldplay",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Parachutes") and eq_ci(args.get("type"), "album")
+        else (False, f"expected query='Parachutes' type='album', got {args}"),
+    ),
+    (
+        "explicit 'track' keyword",
+        "can you play the track Yellow",
+        "search_spotify",
+        lambda args: (True, None) if eq_ci(args.get("query"), "Yellow") and eq_ci(args.get("type"), "track")
+        else (False, f"expected query='Yellow' type='track', got {args}"),
+    ),
+    (
+        "pause",
+        "pause the music",
+        "control_playback",
+        lambda args: (True, None) if args.get("action") == "pause"
+        else (False, f"expected action='pause', got {args}"),
+    ),
+    (
+        "next track",
+        "skip to the next song",
+        "control_playback",
+        lambda args: (True, None) if args.get("action") == "next_track"
+        else (False, f"expected action='next_track', got {args}"),
+    ),
+    (
+        "stop",
+        "stop the music in the living room",
+        "control_playback",
+        lambda args: (True, None) if args.get("action") == "stop"
+        else (False, f"expected action='stop', got {args}"),
+    ),
+    (
+        "shuffle on",
+        "turn on shuffle",
+        "control_playback",
+        lambda args: (True, None) if args.get("action") == "shuffle_on"
+        else (False, f"expected action='shuffle_on', got {args}"),
+    ),
+    (
+        "volume with 'percent' phrasing",
+        "set volume to 80 percent",
+        "control_playback",
+        lambda args: (True, None) if args.get("action") == "volume_set" and float(args.get("volume_level", -1)) == 80
+        else (False, f"expected action='volume_set' volume_level=80, got {args}"),
+    ),
+]
+
+
+def call_model_step2(system_prompt, user_message, tools, step1_name, step1_args):
+    """Feed a fake search_spotify tool result back and get the model's next tool call
+    (should be play_music or queue_music)."""
+    fake_result = {
+        "uri": f"spotify:{step1_args.get('type', 'track')}:FAKEURI0000000000000000",
+        "name": step1_args.get("query", ""),
+        "type": step1_args.get("type", "track"),
+    }
+    tool_call_id = "call_1"
+    messages = [
+        {"role": "system", "content": system_prompt},
+        {"role": "user", "content": user_message},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [{
+                "id": tool_call_id,
+                "type": "function",
+                "function": {"name": step1_name, "arguments": json.dumps(step1_args)},
+            }],
+        },
+        {
+            "role": "tool",
+            "tool_call_id": tool_call_id,
+            "name": step1_name,
+            "content": json.dumps(fake_result),
+        },
+    ]
+    resp = requests.post(
+        f"{OLLAMA_BASE_URL}/chat/completions",
+        headers={"Authorization": f"Bearer {OLLAMA_API_KEY}"},
+        json={"model": MODEL, "messages": messages, "tools": tools, "tool_choice": "auto"},
+        timeout=120,
+    )
+    resp.raise_for_status()
+    return resp.json()
+
+
+def _check_flow_once(user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools):
+    response1 = call_model(system_prompt, user_message, tools)
+    name1, args1, content1 = first_tool_call(response1)
+    if name1 != "search_spotify":
+        return False, f"STEP 1 FAILED — expected search_spotify, got {name1!r} ({args1 or content1})"
+
+    response2 = call_model_step2(system_prompt, user_message, tools, name1, args1)
+    name2, args2, content2 = first_tool_call(response2)
+    if name2 is None:
+        return False, f"step1={name1}({args1}) -> STEP 2 no tool call, model replied with text: {content2!r}"
+
+    detail = f"step1={name1}({args1}) -> step2={name2}({args2})"
+
+    if name2 != expect_step2_name:
+        return False, f"{detail} — WRONG FUNCTION, expected {expect_step2_name!r}"
+    if not eq_ci(args2.get("room"), expect_room):
+        return False, f"{detail} — expected room={expect_room!r}"
+    if not eq_ci(args2.get("type"), expect_type):
+        return False, f"{detail} — expected type={expect_type!r} (must be passed through from step1, not hardcoded)"
+
+    return True, detail
+
+
+def check_flow(label, user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools):
+    """Runs search_spotify -> (fake result) -> play_music/queue_music, repeated REPEAT
+    times, and checks the second call's function name, room, and type parameters."""
+    attempts = [
+        _check_flow_once(user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools)
+        for _ in range(REPEAT)
+    ]
+    passed = sum(1 for ok, _ in attempts if ok)
+    return _summarize(label, user_message, passed, attempts)
+
+
+TESTS_FLOW = [
+    # (label, user_message, expect_step2_name, expect_room, expect_type)
+    (
+        "play + explicit room",
+        "play Fred again in the bedroom",
+        "play_music",
+        "Bedroom",
+        "artist",
+    ),
+    (
+        "queue + artist -> still calls queue_music (fallback handled inside the script now)",
+        "queue Fred again in the bedroom",
+        "queue_music",
+        "Bedroom",
+        "artist",
+    ),
+    (
+        "play + no room specified -> default room",
+        "play Yellow by Coldplay",
+        "play_music",
+        "Living Room",
+        "track",
+    ),
+    (
+        "playlist/genre + explicit room",
+        "play jazz in the bedroom",
+        "play_music",
+        "Bedroom",
+        "playlist",
+    ),
+    (
+        "queue + album -> still calls queue_music (fallback handled inside the script now)",
+        "queue Dark Side of the Moon",
+        "queue_music",
+        "Living Room",
+        "album",
+    ),
+    (
+        "queue + playlist -> still calls queue_music (fallback handled inside the script now)",
+        "queue my workout playlist in the bedroom",
+        "queue_music",
+        "Bedroom",
+        "playlist",
+    ),
+    (
+        "'add to queue' phrasing + artist -> still calls queue_music",
+        "add some Blink 182 to the queue in the living room",
+        "queue_music",
+        "Living Room",
+        "artist",
+    ),
+    (
+        "queue + track, no room specified -> default room, stays queue_music",
+        "queue Yellow by Coldplay",
+        "queue_music",
+        "Living Room",
+        "track",
+    ),
+    (
+        "artist ('music by X' pattern) + explicit room, carried through to step 2",
+        "play some music by Jungle in the bedroom",
+        "play_music",
+        "Bedroom",
+        "artist",
+    ),
+    (
+        "number-titled track + explicit room, carried through to step 2",
+        "play back on 74 in the bedroom",
+        "play_music",
+        "Bedroom",
+        "track",
+    ),
+    (
+        "queue + number-titled track, explicit room",
+        "queue 99 Luftballons in the living room",
+        "queue_music",
+        "Living Room",
+        "track",
+    ),
+]
+
+
+def main():
+    system_prompt = render_prompt()
+    tools = load_tools()
+
+    print(f"Model: {MODEL}")
+    print(f"Endpoint: {OLLAMA_BASE_URL}")
+    print(f"Repeat per test: {REPEAT}")
+    print(f"Loaded {len(tools)} tools: {[t['function']['name'] for t in tools]}")
+
+    results = []
+    for label, user_message, expect_name, expect_fn in TESTS:
+        results.append((label, check(label, user_message, expect_name, expect_fn, system_prompt, tools)))
+
+    print(f"\n{'#' * 40}\n# Two-step flow tests (search -> play/queue)\n{'#' * 40}")
+    for label, user_message, expect_step2_name, expect_room, expect_type in TESTS_FLOW:
+        results.append((label, check_flow(label, user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools)))
+
+    total_passed = sum(passed for _, (passed, n) in results)
+    total_attempts = sum(n for _, (passed, n) in results)
+    reliable = [label for label, (passed, n) in results if passed == n]
+    flaky = [(label, passed, n) for label, (passed, n) in results if 0 < passed < n]
+    failing = [label for label, (passed, n) in results if passed == 0]
+
+    print(f"\n{'=' * 40}")
+    print(f"{total_passed}/{total_attempts} individual attempts passed across {len(results)} tests (x{REPEAT} reps)")
+    print(f"  {len(reliable)}/{len(results)} tests fully reliable (all {REPEAT} reps passed)")
+    if flaky:
+        print(f"  {len(flaky)} FLAKY tests (inconsistent across reps):")
+        for label, passed, n in flaky:
+            print(f"    - {label} ({passed}/{n})")
+    if failing:
+        print(f"  {len(failing)} tests failing every rep:")
+        for label in failing:
+            print(f"    - {label}")
+
+    sys.exit(0 if not flaky and not failing else 1)
+
+
+if __name__ == "__main__":
+    main()