Created
November 27, 2023 19:32
-
-
Save thorhojhus/05cab5667c2e3453e50ce92eaac4d61b to your computer and use it in GitHub Desktop.
Run mistral models (OpenHermes-2.5) with simple code interpreter using ctransformers.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| from ctransformers import AutoModelForCausalLM | |
| from transformers import AutoTokenizer | |
| from colorama import Fore | |
| import torch | |
| from io import StringIO | |
| from contextlib import redirect_stdout | |
| import os, sys, traceback | |
| sys.path.append(os.getcwd()) | |
| tokenizer = AutoTokenizer.from_pretrained("teknium/OpenHermes-2.5-Mistral-7B") | |
| model_path = "C:\models\TheBloke\OpenHermes-2.5-Mistral-7B-GGUF\openhermes-2.5-mistral-7b.Q8_0.gguf" | |
| model = AutoModelForCausalLM.from_pretrained(model_path, gpu_layers=128, threads=1, context_length=4096, max_new_tokens=-1) | |
| system_prompt = "You're Grek. A useful assistant. You use python to solve problems." | |
| def encode_prompt(role, prompt): return tokenizer.encode(f"<|im_start|>{role}\n{prompt}<|im_end|>\n") | |
| def start_prompt(role): return f"<|im_start|>{role}\n" | |
| toks = encode_prompt("system", system_prompt) | |
| prompt = True | |
| outputted = "" | |
| new_toks = [] | |
| while True: | |
| if prompt: | |
| toks += new_toks | |
| user_input = input(Fore.YELLOW + "User: " + Fore.WHITE) | |
| print() | |
| toks += encode_prompt("user", user_input) | |
| toks += tokenizer.encode(start_prompt("assistant")) | |
| print(Fore.CYAN + "Assistant: " + Fore.WHITE, end="") | |
| old_output_len = len(outputted) | |
| new_toks = [] | |
| for tok in model.generate(torch.tensor(toks)): | |
| new_toks += [tok] | |
| char = str(model.detokenize(tok)) | |
| print(char, end="", flush=True) | |
| outputted += char | |
| new_output = outputted[old_output_len:] | |
| if new_output.endswith("```") and '```python\n' in new_output: | |
| if input(Fore.GREEN + "\nDo you want to run this code? (y/n): ").lower() == "y": | |
| python_code = new_output.split('```python\n')[1].split("```")[0] | |
| std_out = StringIO() | |
| try: | |
| with redirect_stdout(std_out): exec(python_code) | |
| result = std_out.getvalue() | |
| except Exception as e: | |
| result = ''.join(traceback.format_exception_only(e)) | |
| python_output = f"\nOutput:\n```\n{result}```" | |
| print(Fore.GREEN + f"\nOutput:\n```\n{result}```") | |
| result_toks = tokenizer.encode(f"\nOutput:\n```\n{result}```") | |
| toks += result_toks | |
| old_output_len = len(outputted) | |
| break | |
| print("\n") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment