diff --git a/current-functions.txt b/current-functions.txt new file mode 100644 index 0000000..357e8f7 --- /dev/null +++ b/current-functions.txt @@ -0,0 +1,225 @@ +- spec: + name: execute_services + description: Use this function to execute service of devices in Home Assistant. + parameters: + type: object + properties: + list: + type: array + items: + type: object + properties: + domain: + type: string + description: The domain of the service + service: + type: string + description: The service to be called + service_data: + type: object + description: The service data object to indicate what to control. + properties: + entity_id: + type: string + description: The entity_id retrieved from available devices. It + must start with domain, followed by dot character. + required: + - entity_id + required: + - domain + - service + - service_data + function: + type: native + name: execute_service + + +# Spotify Music Control Functions + +- spec: + name: search_spotify + description: >- + Search Spotify for an artist, album, track, or playlist and return the Spotify URI. + Use this before playing music to get the URI. Use this whenever the user asks to play + or queue a specific song, artist, album, or playlist — even if the name contains a + number (e.g. "Back on 74", "1979", "Blink 182"). A number that is part of a + song/artist/album name is NOT a volume level; only use control_playback's volume_set + if the user explicitly says "volume" or "percent". + parameters: + type: object + properties: + query: + type: string + description: Artist, album, track, or playlist name to search for (e.g., "Coldplay", "Parachutes", "Yellow", "Today's Top Hits") + type: + type: string + enum: [artist, album, track, playlist] + description: Type of content to search for. Playlist searches check your personal playlists first, then fall back to public Spotify playlists. + required: + - query + - type + function: + type: script + sequence: + - service: spotify_voice_assistant.search + response_variable: _function_result + data: + query: "{{ query }}" + type: "{{ type }}" + +- spec: + name: play_music + description: Play music immediately on a Spotify Connect device using a Spotify URI, replacing current playback. You must first call search_spotify to get the URI. Use this when user says "play". + parameters: + type: object + properties: + spotify_uri: + type: string + description: Spotify URI from search_spotify (e.g., "spotify:artist:4gzpq5DPGxSnKTe4SA8HAU") + type: + type: string + enum: [artist, album, track, playlist] + description: The exact "type" value returned by search_spotify for this URI. Required — passing the wrong value causes playback to silently fail. + room: + type: string + description: Spotify Connect source/device name to play on, matching an entry in media_player.spotify_bhetherman's source_list exactly (e.g., "Living Room") + required: + - spotify_uri + - type + - room + function: + type: script + sequence: + - service: media_player.select_source + target: + entity_id: media_player.spotify_bhetherman + data: + source: "{{ room }}" + - delay: "00:00:02" + - service: media_player.play_media + target: + entity_id: media_player.spotify_bhetherman + data: + media_content_id: "{{ spotify_uri }}" + media_content_type: "{{ type }}" + enqueue: replace + +- spec: + name: queue_music + description: >- + Queue or play music using a Spotify URI from search_spotify. Use this whenever the + user says "queue" or "add to queue" — always use this function for queue requests, + regardless of whether the content is a track, album, artist, or playlist; the + correct behavior for each type is handled automatically. (Individual tracks are + added to the queue without interrupting playback; since Spotify cannot queue a whole + album/artist/playlist, those start playing immediately instead.) + parameters: + type: object + properties: + spotify_uri: + type: string + description: Spotify URI from search_spotify (e.g., "spotify:track:4gzpq5DPGxSnKTe4SA8HAU") + type: + type: string + enum: [artist, album, track, playlist] + description: The exact "type" value returned by search_spotify for this URI. + room: + type: string + description: Spotify Connect source/device name to queue on, matching an entry in media_player.spotify_bhetherman's source_list exactly (e.g., "Living Room") + required: + - spotify_uri + - type + - room + function: + type: script + sequence: + - service: media_player.select_source + target: + entity_id: media_player.spotify_bhetherman + data: + source: "{{ room }}" + - delay: "00:00:02" + - service: media_player.play_media + target: + entity_id: media_player.spotify_bhetherman + data: + media_content_id: "{{ spotify_uri }}" + media_content_type: "{{ type }}" + enqueue: "{{ 'add' if type == 'track' else 'replace' }}" + +- spec: + name: control_playback + description: >- + Control playback of music that is ALREADY PLAYING (pause, resume, stop, next track, + previous track, set volume, shuffle) on the currently active Spotify Connect session. + Only use volume_set when the user explicitly says the word "volume" or "percent" + (e.g. "turn the volume up", "set volume to 50", "50 percent volume"). + Do NOT use this function to play a requested song, artist, or album — a number in a + song/artist/album name (e.g. "Back on 74", "Blink 182", "Sunset 2005") is part of the + title, never a volume level. Use search_spotify + play_music for any request to play + or start specific music, even if it contains a number. + parameters: + type: object + properties: + action: + type: string + enum: [pause, play, stop, next_track, previous_track, volume_set, shuffle_on, shuffle_off] + description: Playback control action + volume_level: + type: number + description: Volume level (0-100) for volume_set action only. Only set this when the user said the word "volume" or "percent". + required: + - action + function: + type: script + sequence: + - choose: + - conditions: "{{ action == 'pause' }}" + sequence: + - service: media_player.media_pause + target: + entity_id: media_player.spotify_bhetherman + - conditions: "{{ action == 'play' }}" + sequence: + - service: media_player.media_play + target: + entity_id: media_player.spotify_bhetherman + - conditions: "{{ action == 'stop' }}" + sequence: + # media_player.spotify_bhetherman does not support the media_stop + # service (STOP is not in its supported_features); Spotify Connect has + # no separate stop concept, so route "stop" through pause instead. + - service: media_player.media_pause + target: + entity_id: media_player.spotify_bhetherman + - conditions: "{{ action == 'next_track' }}" + sequence: + - service: media_player.media_next_track + target: + entity_id: media_player.spotify_bhetherman + - conditions: "{{ action == 'previous_track' }}" + sequence: + - service: media_player.media_previous_track + target: + entity_id: media_player.spotify_bhetherman + - conditions: "{{ action == 'volume_set' }}" + sequence: + - service: media_player.volume_set + target: + entity_id: media_player.spotify_bhetherman + data: + volume_level: "{{ volume_level / 100 }}" + - conditions: "{{ action == 'shuffle_on' }}" + sequence: + - service: media_player.shuffle_set + target: + entity_id: media_player.spotify_bhetherman + data: + shuffle: true + - conditions: "{{ action == 'shuffle_off' }}" + sequence: + - service: media_player.shuffle_set + target: + entity_id: media_player.spotify_bhetherman + data: + shuffle: false \ No newline at end of file diff --git a/current-prompt.txt b/current-prompt.txt new file mode 100644 index 0000000..75b0222 --- /dev/null +++ b/current-prompt.txt @@ -0,0 +1,62 @@ +I want you to act as smart home manager of Home Assistant. +I will provide information of smart home along with a question, you will truthfully make correction or answer using information provided in one sentence in everyday language. + +Current Time: {{now()}} + +Available Devices: +```csv +entity_id,name,state,aliases +{% for entity in exposed_entities -%} +{{ entity.entity_id }},{{ entity.name }},{{ entity.state }},{{entity.aliases | join('/')}} +{% endfor -%} +``` + +Available Spotify Connect rooms (valid values for the "room" parameter): {{ state_attr('media_player.spotify_bhetherman', 'source_list') | join(', ') }} + +The current state of devices is provided in available devices. +Use execute_services function only for requested action, not for current states. +Do not execute service without user's confirmation. Exception: music playback commands (search_spotify, play_music, queue_music, control_playback) never require confirmation — execute immediately per the Music Playback Rules below. +Do not restate or appreciate what user says, rather make a quick inquiry. + +Music Playback Rules: + +CRITICAL: a number inside a song/artist/album name (e.g. "Back on 74", "1979", "Blink 182") is part of the title, NOT a volume level. Only treat a number as a volume level when the user says the word "volume" or "percent". When in doubt between search_spotify and control_playback's volume_set, prefer search_spotify. + +For any request to play or queue music, always call TWO functions in order, with no confirmation: +1. search_spotify — to get the Spotify URI and its type +2. play_music (if the user said "play") or queue_music (if the user said "queue"/"add to queue") + +When calling play_music or queue_music, always set type to the exact "type" value that search_spotify's result returned — never guess or hardcode it. Passing the wrong type silently breaks playback. + +RULE: if the request has the form "[title] by [artist]" (a specific song/album title followed by "by" and an artist name — no matter what words come before it, e.g. "play", "queue", "throw on", "put on", "the album", "the song"), the query is ONLY the title — always strip " by [artist]" from the end. Never include the word "by" or the artist name in the query for this pattern. +- "Play Yellow by Coldplay" → query="Yellow" +- "Throw on the album Parachutes by Coldplay" → query="Parachutes" +- "Queue Wish You Were Here by Pink Floyd" → query="Wish You Were Here" + +How to fill in search_spotify(query, type) — match these examples: +- "Play Coldplay" → query="Coldplay", type="artist" +- "Play songs by Taylor Swift on the kitchen speaker" → query="Taylor Swift", type="artist" +- "Play some music by Jungle in the living room" → query="Jungle", type="artist" +- "Play back on 74 in the living room" → query="Back on 74", type="track" (this is a song title, not a volume command) +- "Play the song Yellow" → query="Yellow", type="track" +- "Play Yellow by Coldplay" → query="Yellow", type="track" +- "Play 1979 by the Smashing Pumpkins" → query="1979", type="track" (a bare number here is a song title, not an album — default to "track" unless the user explicitly says "album") +- "Queue Dark Side of the Moon in the kitchen" → query="Dark Side of the Moon", type="album" +- "Add Wish You Were Here to queue in the bathroom" → query="Wish You Were Here", type="track" +- "Queue Fred again in the bedroom" → query="Fred again", type="artist" (an artist name after "queue" with no specific song/album title named is still type="artist", same as "play") +- "Play my workout playlist" → query="workout", type="playlist" +- "Play jazz" → query="jazz", type="playlist" + +Volume/playback control examples (use control_playback, NOT search_spotify): +- "Set the volume to 50" / "50 percent volume" → action="volume_set", volume_level=50 +- "Turn the volume up" → action="volume_set" +- "Pause the music" → action="pause" +- "Skip this song" → action="next_track" + +Room: +- The "room" parameter for play_music/queue_music must exactly match one of the Available Spotify Connect rooms listed above. +- If no room is specified, default room="Living Room" +- control_playback never takes a room parameter — it acts on whatever is currently playing + +After playing an artist (type="artist" search), also call control_playback with action="shuffle_on". +Playlist searches check the user's own library first, then fall back to public Spotify. \ No newline at end of file diff --git a/test_prompt.py b/test_prompt.py new file mode 100644 index 0000000..e1bfb0e --- /dev/null +++ b/test_prompt.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 +"""Test search_spotify/control_playback tool-call disambiguation directly against +the Ollama endpoint, bypassing Home Assistant entirely. + +Renders current-prompt.txt (with mock HA template values) and current-functions.txt +(as OpenAI-style tool definitions) exactly as Extended OpenAI Conversation would send +them, then checks whether the model picks the right tool + arguments for each test +phrase. Only evaluates the FIRST tool call the model returns per test (i.e. whether +it correctly picks search_spotify vs control_playback and fills in query/type/action) — +it does not simulate the full multi-turn tool-result loop. +""" +import json +import os +import sys +from datetime import datetime + +import requests +import yaml +from jinja2 import Template + +HERE = os.path.dirname(os.path.abspath(__file__)) + +OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://192.168.50.214:11434/v1") +OLLAMA_API_KEY = os.environ.get("OLLAMA_API_KEY", "38ecbb5e71adf1306291e810826d965afe51fa27ff42e60f") +MODEL = os.environ.get("OLLAMA_MODEL") # set this to the exact `ollama list` tag + +MOCK_SOURCE_LIST = ["Living Room", "Web Player (Firefox)", "Bedroom"] + + +def render_prompt(): + with open(os.path.join(HERE, "current-prompt.txt")) as f: + raw = f.read() + + def state_attr(entity_id, attr): + if entity_id == "media_player.spotify_bhetherman" and attr == "source_list": + return MOCK_SOURCE_LIST + return None + + template = Template(raw) + return template.render( + now=lambda: datetime.now(), + exposed_entities=[], # not relevant to these tests; kept empty on purpose + state_attr=state_attr, + ) + + +def load_tools(): + with open(os.path.join(HERE, "current-functions.txt")) as f: + functions = yaml.safe_load(f) + tools = [] + for entry in functions: + spec = entry["spec"] + tools.append({ + "type": "function", + "function": { + "name": spec["name"], + "description": spec["description"], + "parameters": spec["parameters"], + }, + }) + return tools + + +def call_model(system_prompt, user_message, tools): + if not MODEL: + sys.exit("Set OLLAMA_MODEL env var to the exact `ollama list` tag before running.") + + resp = requests.post( + f"{OLLAMA_BASE_URL}/chat/completions", + headers={"Authorization": f"Bearer {OLLAMA_API_KEY}"}, + json={ + "model": MODEL, + "messages": [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_message}, + ], + "tools": tools, + "tool_choice": "auto", + }, + timeout=120, + ) + resp.raise_for_status() + return resp.json() + + +def first_tool_call(response): + message = response["choices"][0]["message"] + tool_calls = message.get("tool_calls") or [] + if not tool_calls: + return None, None, message.get("content") + call = tool_calls[0] + name = call["function"]["name"] + try: + args = json.loads(call["function"]["arguments"]) + except (json.JSONDecodeError, TypeError): + args = call["function"]["arguments"] + return name, args, None + + +REPEAT = int(os.environ.get("TEST_REPEAT", "15")) + + +def _check_once(user_message, expect_name, expect_fn, system_prompt, tools): + response = call_model(system_prompt, user_message, tools) + name, args, content = first_tool_call(response) + + if name is None: + return False, f"NO TOOL CALL — model replied with text instead: {content!r}" + if name != expect_name: + return False, f"WRONG FUNCTION — expected {expect_name!r}, got {name!r} ({args})" + + ok, reason = expect_fn(args) + if ok: + return True, f"{name}({args})" + return False, f"{name}({args}) — {reason}" + + +def _summarize(label, user_message, passed, attempts): + n = len(attempts) + print(f"\n=== {label} ===") + print(f" user: {user_message!r}") + for i, (ok, detail) in enumerate(attempts, 1): + mark = "✅" if ok else "❌" + print(f" [{i}/{n}] {mark} {detail}") + + if passed == n: + print(f" RESULT: {passed}/{n} — reliable pass") + elif passed == 0: + print(f" RESULT: {passed}/{n} — reliable FAIL") + else: + print(f" RESULT: {passed}/{n} — FLAKY (inconsistent across runs)") + return passed, n + + +def check(label, user_message, expect_name, expect_fn, system_prompt, tools): + attempts = [_check_once(user_message, expect_name, expect_fn, system_prompt, tools) for _ in range(REPEAT)] + passed = sum(1 for ok, _ in attempts if ok) + return _summarize(label, user_message, passed, attempts) + + +def eq_ci(a, b): + return isinstance(a, str) and a.strip().lower() == b.strip().lower() + + +def contains_ci(a, b): + return isinstance(a, str) and b.strip().lower() in a.strip().lower() + + +TESTS = [ + ( + "artist-only, generic 'music by X' phrasing", + "play music by Jungle in the living room", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Jungle") and eq_ci(args.get("type"), "artist") + else (False, f"expected query='Jungle' type='artist', got {args}"), + ), + ( + "song title containing a number (previously misread as volume)", + "play back on 74 in the living room", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Back on 74") and eq_ci(args.get("type"), "track") + else (False, f"expected query='Back on 74' type='track', got {args}"), + ), + ( + "specific title + artist (regression check — previously worked)", + "play Yellow by Coldplay", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Yellow") and eq_ci(args.get("type"), "track") + else (False, f"expected query='Yellow' type='track', got {args}"), + ), + ( + "explicit volume command (must NOT be treated as a search)", + "set the volume to 50", + "control_playback", + lambda args: (True, None) if args.get("action") == "volume_set" and float(args.get("volume_level", -1)) == 50 + else (False, f"expected action='volume_set' volume_level=50, got {args}"), + ), + ( + "album title, different room", + "queue Dark Side of the Moon in the bedroom", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Dark Side of the Moon") and args.get("type") in ("album", "track") + else (False, f"expected query='Dark Side of the Moon' type in (album,track), got {args}"), + ), + ( + "genre/mood, no artist", + "play some jazz", + "search_spotify", + lambda args: (True, None) if contains_ci(args.get("query"), "jazz") and eq_ci(args.get("type"), "playlist") + else (False, f"expected query~'jazz' type='playlist', got {args}"), + ), + ( + "named playlist phrase", + "play my daily mix", + "search_spotify", + lambda args: (True, None) if contains_ci(args.get("query"), "daily mix") and eq_ci(args.get("type"), "playlist") + else (False, f"expected query~'daily mix' type='playlist', got {args}"), + ), + ( + "number-titled track + explicit artist (by-stripping regression variant)", + "play 1979 by the Smashing Pumpkins", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "1979") and eq_ci(args.get("type"), "track") + else (False, f"expected query='1979' type='track', got {args}"), + ), + ( + "number-titled track, no artist given", + "play 99 Luftballons", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "99 Luftballons") and eq_ci(args.get("type"), "track") + else (False, f"expected query='99 Luftballons' type='track', got {args}"), + ), + ( + "artist name that itself contains a number", + "queue some Blink 182", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Blink 182") and eq_ci(args.get("type"), "artist") + else (False, f"expected query='Blink 182' type='artist', got {args}"), + ), + ( + "colloquial phrasing ('put on')", + "put on some Fred again", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Fred again") and eq_ci(args.get("type"), "artist") + else (False, f"expected query='Fred again' type='artist', got {args}"), + ), + ( + "explicit 'album' keyword + by-stripping", + "throw on the album Parachutes by Coldplay", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Parachutes") and eq_ci(args.get("type"), "album") + else (False, f"expected query='Parachutes' type='album', got {args}"), + ), + ( + "explicit 'track' keyword", + "can you play the track Yellow", + "search_spotify", + lambda args: (True, None) if eq_ci(args.get("query"), "Yellow") and eq_ci(args.get("type"), "track") + else (False, f"expected query='Yellow' type='track', got {args}"), + ), + ( + "pause", + "pause the music", + "control_playback", + lambda args: (True, None) if args.get("action") == "pause" + else (False, f"expected action='pause', got {args}"), + ), + ( + "next track", + "skip to the next song", + "control_playback", + lambda args: (True, None) if args.get("action") == "next_track" + else (False, f"expected action='next_track', got {args}"), + ), + ( + "stop", + "stop the music in the living room", + "control_playback", + lambda args: (True, None) if args.get("action") == "stop" + else (False, f"expected action='stop', got {args}"), + ), + ( + "shuffle on", + "turn on shuffle", + "control_playback", + lambda args: (True, None) if args.get("action") == "shuffle_on" + else (False, f"expected action='shuffle_on', got {args}"), + ), + ( + "volume with 'percent' phrasing", + "set volume to 80 percent", + "control_playback", + lambda args: (True, None) if args.get("action") == "volume_set" and float(args.get("volume_level", -1)) == 80 + else (False, f"expected action='volume_set' volume_level=80, got {args}"), + ), +] + + +def call_model_step2(system_prompt, user_message, tools, step1_name, step1_args): + """Feed a fake search_spotify tool result back and get the model's next tool call + (should be play_music or queue_music).""" + fake_result = { + "uri": f"spotify:{step1_args.get('type', 'track')}:FAKEURI0000000000000000", + "name": step1_args.get("query", ""), + "type": step1_args.get("type", "track"), + } + tool_call_id = "call_1" + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_message}, + { + "role": "assistant", + "content": None, + "tool_calls": [{ + "id": tool_call_id, + "type": "function", + "function": {"name": step1_name, "arguments": json.dumps(step1_args)}, + }], + }, + { + "role": "tool", + "tool_call_id": tool_call_id, + "name": step1_name, + "content": json.dumps(fake_result), + }, + ] + resp = requests.post( + f"{OLLAMA_BASE_URL}/chat/completions", + headers={"Authorization": f"Bearer {OLLAMA_API_KEY}"}, + json={"model": MODEL, "messages": messages, "tools": tools, "tool_choice": "auto"}, + timeout=120, + ) + resp.raise_for_status() + return resp.json() + + +def _check_flow_once(user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools): + response1 = call_model(system_prompt, user_message, tools) + name1, args1, content1 = first_tool_call(response1) + if name1 != "search_spotify": + return False, f"STEP 1 FAILED — expected search_spotify, got {name1!r} ({args1 or content1})" + + response2 = call_model_step2(system_prompt, user_message, tools, name1, args1) + name2, args2, content2 = first_tool_call(response2) + if name2 is None: + return False, f"step1={name1}({args1}) -> STEP 2 no tool call, model replied with text: {content2!r}" + + detail = f"step1={name1}({args1}) -> step2={name2}({args2})" + + if name2 != expect_step2_name: + return False, f"{detail} — WRONG FUNCTION, expected {expect_step2_name!r}" + if not eq_ci(args2.get("room"), expect_room): + return False, f"{detail} — expected room={expect_room!r}" + if not eq_ci(args2.get("type"), expect_type): + return False, f"{detail} — expected type={expect_type!r} (must be passed through from step1, not hardcoded)" + + return True, detail + + +def check_flow(label, user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools): + """Runs search_spotify -> (fake result) -> play_music/queue_music, repeated REPEAT + times, and checks the second call's function name, room, and type parameters.""" + attempts = [ + _check_flow_once(user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools) + for _ in range(REPEAT) + ] + passed = sum(1 for ok, _ in attempts if ok) + return _summarize(label, user_message, passed, attempts) + + +TESTS_FLOW = [ + # (label, user_message, expect_step2_name, expect_room, expect_type) + ( + "play + explicit room", + "play Fred again in the bedroom", + "play_music", + "Bedroom", + "artist", + ), + ( + "queue + artist -> still calls queue_music (fallback handled inside the script now)", + "queue Fred again in the bedroom", + "queue_music", + "Bedroom", + "artist", + ), + ( + "play + no room specified -> default room", + "play Yellow by Coldplay", + "play_music", + "Living Room", + "track", + ), + ( + "playlist/genre + explicit room", + "play jazz in the bedroom", + "play_music", + "Bedroom", + "playlist", + ), + ( + "queue + album -> still calls queue_music (fallback handled inside the script now)", + "queue Dark Side of the Moon", + "queue_music", + "Living Room", + "album", + ), + ( + "queue + playlist -> still calls queue_music (fallback handled inside the script now)", + "queue my workout playlist in the bedroom", + "queue_music", + "Bedroom", + "playlist", + ), + ( + "'add to queue' phrasing + artist -> still calls queue_music", + "add some Blink 182 to the queue in the living room", + "queue_music", + "Living Room", + "artist", + ), + ( + "queue + track, no room specified -> default room, stays queue_music", + "queue Yellow by Coldplay", + "queue_music", + "Living Room", + "track", + ), + ( + "artist ('music by X' pattern) + explicit room, carried through to step 2", + "play some music by Jungle in the bedroom", + "play_music", + "Bedroom", + "artist", + ), + ( + "number-titled track + explicit room, carried through to step 2", + "play back on 74 in the bedroom", + "play_music", + "Bedroom", + "track", + ), + ( + "queue + number-titled track, explicit room", + "queue 99 Luftballons in the living room", + "queue_music", + "Living Room", + "track", + ), +] + + +def main(): + system_prompt = render_prompt() + tools = load_tools() + + print(f"Model: {MODEL}") + print(f"Endpoint: {OLLAMA_BASE_URL}") + print(f"Repeat per test: {REPEAT}") + print(f"Loaded {len(tools)} tools: {[t['function']['name'] for t in tools]}") + + results = [] + for label, user_message, expect_name, expect_fn in TESTS: + results.append((label, check(label, user_message, expect_name, expect_fn, system_prompt, tools))) + + print(f"\n{'#' * 40}\n# Two-step flow tests (search -> play/queue)\n{'#' * 40}") + for label, user_message, expect_step2_name, expect_room, expect_type in TESTS_FLOW: + results.append((label, check_flow(label, user_message, expect_step2_name, expect_room, expect_type, system_prompt, tools))) + + total_passed = sum(passed for _, (passed, n) in results) + total_attempts = sum(n for _, (passed, n) in results) + reliable = [label for label, (passed, n) in results if passed == n] + flaky = [(label, passed, n) for label, (passed, n) in results if 0 < passed < n] + failing = [label for label, (passed, n) in results if passed == 0] + + print(f"\n{'=' * 40}") + print(f"{total_passed}/{total_attempts} individual attempts passed across {len(results)} tests (x{REPEAT} reps)") + print(f" {len(reliable)}/{len(results)} tests fully reliable (all {REPEAT} reps passed)") + if flaky: + print(f" {len(flaky)} FLAKY tests (inconsistent across reps):") + for label, passed, n in flaky: + print(f" - {label} ({passed}/{n})") + if failing: + print(f" {len(failing)} tests failing every rep:") + for label in failing: + print(f" - {label}") + + sys.exit(0 if not flaky and not failing else 1) + + +if __name__ == "__main__": + main()