Created
December 5, 2025 04:44
-
-
Save coverband/a6839177e2cbcb3e735bd4a2efaa2b90 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # --------------------------------------------- | |
| # This is another small python utility that will get a list of all blackbox.ai models that are supposed to be accessible to you, | |
| # and do a quick test call with each to figure out which ones ACTUALLY work. Many models are listed by blackbox.ai, but unfortunately | |
| # a large number of them are not actually accessible, some due to API rate throttling by the provider, some due to the provider asking | |
| # you to accept their terms/policies before granting access, and some due to incorrect path/documentation/version by blackbox.ai. | |
| # This version of the script checks to see if the model supports "tools" parameter, since that's what gets added to the query by the | |
| # CLI version of blackbox.ai | |
| # --------------------------------------------- | |
| import requests | |
| import json | |
| import time | |
| # --------------------------------------------- | |
| # CONFIG | |
| # --------------------------------------------- | |
| BLACKBOX_API_KEY = "YOUR_BLACKBOX_API_KEY_HERE" | |
| BASE_URL = "https://api.blackbox.ai/v1" | |
| TIMEOUT = 15 # seconds per request | |
| PAUSE_BETWEEN_CALLS = 0.4 # small delay to avoid rate limits | |
| # A simple test tool definition | |
| TEST_TOOLS = [ | |
| { | |
| "type": "function", | |
| "function": { | |
| "name": "ping", | |
| "description": "Simple tool to test tool-calling support.", | |
| "parameters": { | |
| "type": "object", | |
| "properties": { | |
| "message": { | |
| "type": "string", | |
| "description": "Message to echo back" | |
| } | |
| }, | |
| "required": ["message"] | |
| } | |
| } | |
| } | |
| ] | |
| TEST_MESSAGE = ( | |
| "If you can call tools, call the 'ping' tool with message='tool test'. " | |
| "If you cannot call tools, just reply 'NO_TOOL'." | |
| ) | |
| # --------------------------------------------- | |
| headers = { | |
| "Authorization": f"Bearer {BLACKBOX_API_KEY}", | |
| "Content-Type": "application/json", | |
| } | |
| def get_models(): | |
| """Fetch the list of available models from Blackbox /v1/models.""" | |
| print("Fetching model list...\n") | |
| url = f"{BASE_URL}/models" | |
| try: | |
| resp = requests.get(url, headers=headers, timeout=TIMEOUT) | |
| except Exception as e: | |
| print("ERROR requesting /models:", e) | |
| return [] | |
| if resp.status_code != 200: | |
| print("ERROR retrieving models:", resp.status_code, resp.text) | |
| return [] | |
| data = resp.json() | |
| # Blackbox / LiteLLM can use "data" or "models" as the list key | |
| models = data.get("data") or data.get("models") or [] | |
| print(f"Found {len(models)} models.\n") | |
| return models | |
| def test_model_with_tools(model_id): | |
| """ | |
| Send a small test message to a given model using tools + tool_choice. | |
| Returns (success: bool, result_or_error: str|dict). | |
| """ | |
| url = f"{BASE_URL}/chat/completions" | |
| payload = { | |
| "model": model_id, | |
| "messages": [ | |
| {"role": "user", "content": TEST_MESSAGE} | |
| ], | |
| "tools": TEST_TOOLS, | |
| # We just need the API to accept tools; 'auto' is usually safest | |
| "tool_choice": "auto", | |
| } | |
| try: | |
| resp = requests.post(url, headers=headers, json=payload, timeout=TIMEOUT) | |
| except Exception as e: | |
| return False, f"Request error: {e}" | |
| if resp.status_code == 200: | |
| # You can inspect resp.json() if you want to see how it used tools | |
| return True, resp.json() | |
| else: | |
| # Return error info for logging; this is where 404 / 'No endpoints found' will show | |
| try: | |
| return False, f"HTTP {resp.status_code}: {resp.text}" | |
| except Exception: | |
| return False, f"HTTP {resp.status_code} (non-text body)" | |
| def main(): | |
| models = get_models() | |
| if not models: | |
| print("No models found — stopping.") | |
| return | |
| working_with_tools = [] | |
| failing_with_tools = [] | |
| for m in models: | |
| # LiteLLM / Blackbox may store model identifier under "id" or "name" | |
| model_id = m.get("id") or m.get("name") | |
| if not model_id: | |
| continue | |
| print(f"Testing tools support for model: {model_id} ...") | |
| success, result = test_model_with_tools(model_id) | |
| if success: | |
| print(f" ✔ SUCCESS (tools request accepted)") | |
| working_with_tools.append(model_id) | |
| else: | |
| print(f" ✖ FAILED (tools request) — {result}") | |
| failing_with_tools.append((model_id, result)) | |
| time.sleep(PAUSE_BETWEEN_CALLS) | |
| # --------------------------------------------- | |
| # Summary | |
| # --------------------------------------------- | |
| print("\n\n=========== TOOL SUPPORT TEST SUMMARY ===========\n") | |
| print("✔ Models that accepted a tools-enabled request (HTTP 200):") | |
| for w in working_with_tools: | |
| print(" -", w) | |
| print("\n✖ Models that failed with tools (non-200 / errors):") | |
| for f, err in failing_with_tools: | |
| print(f" - {f}: {err}") | |
| print("\n=================================================\n") | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment