Skip to content

Instantly share code, notes, and snippets.

@kishida

kishida/java-error-gptoss-20b.ipynb

Created November 30, 2025 13:24
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save kishida/c0ea37294a983b7a930caaac828fc742 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save kishida/c0ea37294a983b7a930caaac828fc742 to your computer and use it in GitHub Desktop.
Download ZIP
Javaのエラーを明るく解説するgpt-oss 20b Fine Tune
Display the source blob
Display the rendered blob
Raw
java-error-gptoss-20b.ipynb
{
"cells": [
{
"cell_type": "markdown",
"id": "a8d5bf55-2da8-4545-81bc-be1119e61f78",
"metadata": {
"editable": true,
"slideshow": {
"slide_type": ""
},
"tags": []
},
"source": [
"## 設定"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df7c3391-e202-41a9-b206-2985f80b8e57",
"metadata": {},
"outputs": [],
"source": [
"max_seq_length = 2048\n",
"model_name = \"unsloth/gpt-oss-20b\"\n",
"max_steps = 30 # need 150 or higher to get the effect?\n",
"use_wandb = True\n",
"output_dir = \"outputs_gptoss\"\n",
"save_lora = False # save to output_dir + \"_lora\"\n",
"save_merged = True # save to output_dir + \"_merged\"\n",
"save_gguf = False\n",
"push_gguf = False\n",
"hf_token = \"\"\n",
"\n",
"if push_gguf: save_gguf = False # it will be saved during pushing to hf"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3c9db459-f107-4dd7-9697-d29ab64d9453",
"metadata": {},
"outputs": [],
"source": [
"report_to = \"none\"\n",
"if use_wandb:\n",
" import wandb\n",
" report_to = \"wandb\"\n",
" wandb.login()"
]
},
{
"cell_type": "markdown",
"id": "600b2552-3542-4c36-b279-b96294a488ab",
"metadata": {},
"source": [
"## モジュールインストール"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5ce7c931-0d5c-4e9f-861a-19efa38c1889",
"metadata": {},
"outputs": [],
"source": [
"%%capture\n",
"import os, re\n",
"if \"COLAB_\" in \"\".join(os.environ.keys()):\n",
" # Do this only in Colab notebooks! Otherwise use pip install unsloth\n",
" import torch; v = re.match(r\"[0-9]{1,}\\.[0-9]{1,}\", str(torch.__version__)).group(0)\n",
" xformers = \"xformers==\" + (\"0.0.33.post1\" if v==\"2.9\" else \"0.0.32.post2\" if v==\"2.8\" else \"0.0.29.post3\")\n",
" !pip install --no-deps bitsandbytes accelerate {xformers} peft trl triton cut_cross_entropy unsloth_zoo\n",
" !pip install sentencepiece protobuf \"datasets==4.3.0\" \"huggingface_hub>=0.34.0\" hf_transfer\n",
" !pip install --no-deps unsloth\n",
" !pip install transformers==4.57.1\n",
" !pip install --no-deps trl==0.22.2"
]
},
{
"cell_type": "markdown",
"id": "f4864526-e082-44b6-bbe0-7e9ceab526b7",
"metadata": {},
"source": [
"## モデルの準備"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6296d8ac-dbca-4c9c-bbd0-41ebb075e467",
"metadata": {},
"outputs": [],
"source": [
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name = model_name, \n",
" max_seq_length = max_seq_length,\n",
" dtype = None,\n",
" load_in_4bit = True,\n",
" full_finetuning = False,\n",
" # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "73711cac-2230-4ec5-95c7-4fc4c791769d",
"metadata": {},
"outputs": [],
"source": [
"model = FastLanguageModel.get_peft_model(\n",
" model,\n",
" r = 32, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n",
" target_modules = [\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n",
" \"gate_proj\", \"up_proj\", \"down_proj\",],\n",
" lora_alpha = 32,\n",
" lora_dropout = 0, # Supports any, but = 0 is optimized\n",
" bias = \"none\", # Supports any, but = \"none\" is optimized\n",
" # [NEW] \"unsloth\" uses 30% less VRAM, fits 2x larger batch sizes!\n",
" use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n",
" random_state = 3407,\n",
" use_rslora = False, # We support rank stabilized LoRA\n",
" loftq_config = None, # And LoftQ\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "77765bd3-d775-49e5-8160-79caa8e65101",
"metadata": {},
"outputs": [],
"source": [
"#from unsloth.chat_templates import get_chat_template\n",
"#tokenizer = get_chat_template(\n",
"# tokenizer,\n",
"# chat_template = chat_template,\n",
"#)"
]
},
{
"cell_type": "markdown",
"id": "9b30812f-5fb4-4202-b035-a9bbed0a3758",
"metadata": {},
"source": [
"## データの準備"
]
},
{
"cell_type": "markdown",
"id": "e40e7dda-5640-4853-bc15-e5f1602379b5",
"metadata": {},
"source": [
"### 読み込み"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3bdef308-62a8-4e8b-822b-4808254d8bda",
"metadata": {},
"outputs": [],
"source": [
"from datasets import load_dataset\n",
"dataset = load_dataset(\"kishida/CompileError-Java-JP-cheerful\", split = \"train\")\n",
"filtered = dataset.filter(lambda x: x[\"description\"] is not None)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8f0457e5-51f0-48a1-b2cd-6ce663de4251",
"metadata": {},
"outputs": [],
"source": [
"print(len(dataset))\n",
"print(len(filtered))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b8e8dfeb-fe34-4dd1-8972-2c26ed02a1f0",
"metadata": {},
"outputs": [],
"source": [
"filtered[3]"
]
},
{
"cell_type": "markdown",
"id": "4530e85c-c9af-4110-bdac-0dc1fa29b84e",
"metadata": {},
"source": [
"### 変換"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1bca54d4-85fe-4fc3-98e1-7dbe780c179b",
"metadata": {},
"outputs": [],
"source": [
"template = \"\"\"filename: {}\n",
"## source\n",
"{}\n",
"\n",
"## error message\n",
"{}\n",
"\"\"\"\n",
"system_prompt = \"\"\"You are a good Java error explainer.\n",
"Please provide a concise explanation of the compilation error that occurred in the given source code.\n",
"Do not provide the complete code you have fixed.\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "696095de-afba-4e85-bcaa-5d23366e6b89",
"metadata": {},
"outputs": [],
"source": [
"def make_conv(example):\n",
" prompt = template.format(example[\"filename\"], example[\"code\"], example[\"compile_message\"])\n",
" return {\"conversations\": [\n",
" {\"content\": system_prompt, \"role\": \"system\"},\n",
" {\"content\": prompt, \"role\": \"user\"},\n",
" {\"content\": example[\"description\"], \"role\": \"assistant\"}]}\n",
"dataset = filtered.map(make_conv, batched = False,)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3b6836d5-a354-49f2-8c77-00fc2e3faa26",
"metadata": {},
"outputs": [],
"source": [
"def formatting_prompts_func(examples):\n",
" convos = examples[\"conversations\"]\n",
" texts = [tokenizer.apply_chat_template(conv, tokenize = False, add_generation_prompt = False) for conv in convos]\n",
" return {\"text\": texts, }\n",
"dataset = dataset.map(formatting_prompts_func, batched = True,)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b7093231-40ef-4235-9bbb-1441e03d6b2c",
"metadata": {},
"outputs": [],
"source": [
"dataset[3][\"text\"]"
]
},
{
"cell_type": "markdown",
"id": "d38304a8-b44f-42f8-aa23-caccf3523a11",
"metadata": {},
"source": [
"## 学習"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b78eeff9-1c3a-4cd9-bf8a-6837a14f338d",
"metadata": {},
"outputs": [],
"source": [
"from trl import SFTConfig, SFTTrainer\n",
"from transformers import DataCollatorForSeq2Seq\n",
"trainer = SFTTrainer(\n",
" model = model,\n",
" tokenizer = tokenizer,\n",
" train_dataset = dataset,\n",
" max_seq_length = max_seq_length,\n",
" data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer),\n",
" packing = False, # Can make training 5x faster for short sequences.\n",
" args = SFTConfig(\n",
" dataset_text_field = \"text\",\n",
" per_device_train_batch_size = 2,\n",
" gradient_accumulation_steps = 4,\n",
" warmup_steps = 5,\n",
" # num_train_epochs = 1, # Set this for 1 full training run.\n",
" max_steps = max_steps,\n",
" learning_rate = 2e-4,\n",
" logging_steps = 1,\n",
" optim = \"adamw_8bit\",\n",
" weight_decay = 0.001,\n",
" lr_scheduler_type = \"linear\",\n",
" seed = 3407,\n",
" output_dir = output_dir,\n",
" report_to = report_to, # Use TrackIO/WandB etc\n",
" ),\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ed5c3d82-1a7e-4b66-96b1-74fb8ee808e0",
"metadata": {},
"outputs": [],
"source": [
"tokenizer.decode(trainer.train_dataset[3][\"input_ids\"])"
]
},
{
"cell_type": "markdown",
"id": "9e78cf40-fb12-42dd-8059-2b3b3f0bb488",
"metadata": {},
"source": [
"### 学習開始"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "764f3e5d-72be-4367-8c99-d30712ff12fc",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"# @title Show current memory stats\n",
"gpu_stats = torch.cuda.get_device_properties(0)\n",
"start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n",
"max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n",
"print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n",
"print(f\"{start_gpu_memory} GB of memory reserved.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3632ac67-9699-48c9-b8af-b1337de64d08",
"metadata": {},
"outputs": [],
"source": [
"trainer_stats = trainer.train()\n",
"if use_wandb: wandb.finish()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0d87c1e1-6e42-4e84-8e5e-600ef6fafad5",
"metadata": {},
"outputs": [],
"source": [
"# @title Show final memory and time stats\n",
"used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n",
"used_memory_for_lora = round(used_memory - start_gpu_memory, 3)\n",
"used_percentage = round(used_memory / max_memory * 100, 3)\n",
"lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)\n",
"print(f\"{trainer_stats.metrics['train_runtime']} seconds used for training.\")\n",
"print(\n",
" f\"{round(trainer_stats.metrics['train_runtime']/60, 2)} minutes used for training.\"\n",
")\n",
"print(f\"Peak reserved memory = {used_memory} GB.\")\n",
"print(f\"Peak reserved memory for training = {used_memory_for_lora} GB.\")\n",
"print(f\"Peak reserved memory % of max memory = {used_percentage} %.\")\n",
"print(f\"Peak reserved memory for training % of max memory = {lora_percentage} %.\")"
]
},
{
"cell_type": "markdown",
"id": "d659802f-6623-4899-9335-fd44db303421",
"metadata": {},
"source": [
"## 推論"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fdfa15af-b13c-4789-a1f1-e84c8fb39ac3",
"metadata": {},
"outputs": [],
"source": [
"sample = dataset[3]\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": system_prompt},\n",
" {\"role\": \"user\", \"content\": template.format(sample[\"filename\"], sample[\"code\"], sample[\"compile_message\"])}]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5e12a6af-79a7-4c83-aede-abc1a8a93b94",
"metadata": {},
"outputs": [],
"source": [
"sample[\"description\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "936d1ff9-fbed-4682-9ed7-60703daacec0",
"metadata": {},
"outputs": [],
"source": [
"FastLanguageModel.for_inference(model) # Enable native 2x faster inference\n",
"inputs = tokenizer.apply_chat_template(\n",
" messages,\n",
" # tokenize = True,\n",
" add_generation_prompt = True, # Must add for generation\n",
" return_tensors = \"pt\",\n",
" # return_dict = True,\n",
" # reasoning_effort = \"medium\",\n",
").to(\"cuda\")\n",
"from transformers import TextStreamer\n",
"text_streamer = TextStreamer(tokenizer, skip_prompt = True)\n",
"_ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 256,\n",
" use_cache = True, temperature = 1.5, min_p = 0.1)"
]
},
{
"cell_type": "markdown",
"id": "2ccf3578-8fdf-4fb0-8893-5244ca92949b",
"metadata": {},
"source": [
"## 保存"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ed39e906-d48f-401f-bc62-c5c1a83edadc",
"metadata": {},
"outputs": [],
"source": [
"if save_lora:\n",
" model.save_pretrained(output_dir + \"_lora\") # Local saving\n",
" tokenizer.save_pretrained(output_dir + \"_lora\")\n",
"if save_merged:\n",
" model.save_pretrained_merged(output_dir + \"_merged\", tokenizer,)\n",
"if save_gguf:\n",
" model.save_pretrained_gguf(output_dir + \"_gguf\", tokenizer, quantization_method = \"q4_k_m\")\n",
"if push_gguf:\n",
" model.push_to_hub_gguf(\n",
" output_dir + \"_gguf\", tokenizer, quantization_method = \"q4_k_m\", token = hf_token)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "95a34ebb-f127-48e2-b33b-53c3c9fe3a1c",
"metadata": {},
"outputs": [],
"source": [
"%ls"
]
}
],
"metadata": {
"colab": {
"gpuType": "T4",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment