Last active
June 16, 2025 18:58
-
-
Save yash98/15753a7f2df1368a2e53825d37cb434b to your computer and use it in GitHub Desktop.
How to use Open Router ( openrouter.ai ) with DSPy
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import requests | |
| from os import getenv | |
| from dsp import LM | |
| from ratelimit import limits | |
| RL_CALLS=40 | |
| RL_PERIOD_SECONDS=60 | |
| class OpenRouterClient(LM): | |
| def __init__(self, api_key=None, base_url="https://openrouter.ai/api/v1", model="meta-llama/llama-3-8b-instruct:free", extra_headers=None, **kwargs): | |
| self.api_key = api_key or getenv("OPENROUTER_API_KEY") | |
| self.base_url = base_url | |
| self.model = model | |
| self.extra_headers = extra_headers or {} | |
| self.history = [] | |
| self.provider = "openai" | |
| self.kwargs = {'temperature': 0.0, | |
| 'max_tokens': 150, | |
| 'top_p': 1, | |
| 'frequency_penalty': 0, | |
| 'presence_penalty': 0, | |
| 'n': 1} | |
| self.kwargs.update(kwargs) | |
| def _get_choice_text(choice): | |
| return choice["message"]["content"] | |
| def _get_headers(self): | |
| headers = { | |
| "Authorization": f"Bearer {self.api_key}", | |
| "Content-Type": "application/json" | |
| } | |
| headers.update(self.extra_headers) | |
| return headers | |
| @limits(calls=RL_CALLS, period=RL_PERIOD_SECONDS) | |
| def basic_request(self, prompt: str, **kwargs): | |
| headers = self._get_headers() | |
| data = { | |
| "model": self.model, | |
| "messages": [ | |
| {"role": "user", "content": prompt} | |
| ], | |
| **kwargs | |
| } | |
| response = requests.post(f"{self.base_url}/chat/completions", headers=headers, json=data) | |
| response_data = response.json() | |
| print(response_data) | |
| self.history.append({ | |
| "prompt": prompt, | |
| "response": response_data, | |
| "kwargs": kwargs, | |
| }) | |
| return response_data | |
| def __call__(self, prompt, **kwargs): | |
| req_kwargs = self.kwargs | |
| if not kwargs: | |
| req_kwargs.update(kwargs) | |
| response_data = self.basic_request(prompt, **req_kwargs) | |
| completions = [choice["message"]["content"] for choice in response_data.get("choices", [])] | |
| return completions | |
| if __name__ == "__main__": | |
| # Set the environment variable | |
| os.environ['OPENROUTER_API_KEY'] = '<YOUR KEY>' | |
| turbo = OpenRouterClient() | |
| turbo("hi") | |
| # {'id': 'gen-UPzFCqzOZTRhCteAZ1wdKMeLMri4', 'model': 'meta-llama/llama-3-8b-instruct:free', 'object': 'chat.completion', 'created': 1716115763, 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': "Hi! It's nice to meet you. Is there something I can help you with, or would you like to chat?"}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 43, 'completion_tokens': 25, 'total_tokens': 68}} | |
| # ["Hi! It's nice to meet you. Is there something I can help you with, or would you like to chat?"] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Not able to use with DSPY - Submitted issue here stanfordnlp/dspy#2035