Created
February 11, 2025 18:26
-
-
Save rileyhilliard/735aefc9744d05e633c23970b3ed53dd to your computer and use it in GitHub Desktop.
Example WebAI LLM Chatbot interaction via python in the terminal
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This module demonstrates that it's possible to leverage WebAI to host an LLM chatbot, | |
| # interacting with it via APIs. The code sends a prompt to the LLM and streams back its response. | |
| # In theory, this approach can be extended to programmatically interact with the LLM, | |
| # while using WebAI to orchestrate running the LLM across a cluster of local hardware. | |
| # Demo: https://i.ibb.co/jkNF488Z/Feb-11-2025-12-17-22.gif | |
| import json | |
| import requests | |
| import time | |
| def get_endpoints(): | |
| print("Please enter the endpoint URLs:") | |
| prompt_url = input("Prompt URL (e.g., http://127.0.0.1:10501/prompt): ").strip() | |
| response_url = input("Response URL (e.g., http://127.0.0.1:10502/response): ").strip() | |
| if not prompt_url or not response_url: | |
| raise ValueError("Both Prompt URL and Response URL are required") | |
| return prompt_url, response_url | |
| def fetch_llm_response(chat_history, prompt_url, response_url) -> str: | |
| headers = { | |
| "Content-Type": "application/json", | |
| "Accept": "application/json", | |
| } | |
| data = {"message": chat_history[-1]["content"]} | |
| prompt_response = requests.post(prompt_url, headers=headers, json=data) | |
| if prompt_response.status_code != 200: | |
| print(f"Error: Failed to send prompt. Status code: {prompt_response.status_code}") | |
| return | |
| output = "" | |
| while True: | |
| response = requests.post(response_url, headers=headers) | |
| if response.status_code != 200: | |
| print(f"Error: Failed to fetch response. Status code: {response.status_code}") | |
| return | |
| try: | |
| response_data = response.json() | |
| for item in response_data: | |
| if "message" in item: | |
| print(item["message"], end="", flush=True) | |
| output += item["message"] | |
| if item.get("done", False): | |
| print() | |
| return output | |
| except json.JSONDecodeError: | |
| continue | |
| time.sleep(0.1) | |
| chat_history = [ | |
| { | |
| "role": "system", | |
| "content": "You are a helpful assistant." | |
| }, | |
| ] | |
| if __name__ == "__main__": | |
| prompt_url, response_url = get_endpoints() | |
| while True: | |
| prompt = input("Enter your LLM prompt (or 'exit' to quit): ") | |
| if prompt == "exit": | |
| break | |
| chat_history.append({"role": "user", "content": prompt}) | |
| msg = fetch_llm_response(chat_history, prompt_url, response_url) | |
| if msg: | |
| chat_history.append({"role": "assistant", "content": msg}) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment