Skip to content

Instantly share code, notes, and snippets.

View macleginn's full-sized avatar

Dmitry Nikolayev macleginn

42 followers · 22 following
View GitHub Profile
@macleginn
macleginn / unfold_as_loops.py
Created March 6, 2026 12:27
"""
Conceptual pseudocode explaining how sliding-window attention works
using loops. This is NOT meant to be efficient or runnable in a real
model. It simply illustrates what PyTorch operations like:
unfold
unsqueeze
squeeze
batched matrix multiplication
@macleginn
macleginn / gist:46b41588d48c5c8f7190cb03a3cab0b2
Created March 5, 2026 15:39
mfw_3000.txt
a
abandon
ability
able
abortion
about
above
abroad
absence
absolute
@macleginn
macleginn / dummy_counter_w_main.py
Created November 12, 2025 10:46
class Counter:
def __init__(self):
self.count = 0
def get_value(self):
return self.count
def increment(self):
self.count += 1
@macleginn
macleginn / dummy_counter_as_script.py
Last active November 12, 2025 10:46
class Counter:
def __init__(self):
self.count = 0
def get_value(self):
return self.count
def increment(self):
self.count += 1
@macleginn
macleginn / dummy_counter.py
Created November 12, 2025 10:08
class Counter:
def __init__(self):
self.count = 0
def get_value(self):
return self.count
def increment(self):
self.count += 1
@macleginn
macleginn / dolma_sum_probability.py
Created July 9, 2025 11:31
Estimate the sum probability mass of a corpus using importance sampling
import json
import random
from math import ceil
import numpy as np
import torch
import torch.nn.functional as F
from transformers import AutoModelForCausalLM, AutoTokenizer
from tqdm.auto import tqdm
@macleginn
macleginn / fol_parser.py
Created July 1, 2025 10:21
A parser for FOL expressions equipped with node-delimiting braces and labels
import re
from typing import Optional, Tuple, List, Any
class TreeNode:
"""Represents a node in the parse tree."""
def __init__(self, rule_name: str, content: Any = None, children: List['TreeNode'] = None):
self.rule_name = rule_name
self.content = content # For terminal nodes
self.children = children or []
@macleginn
macleginn / extract_speeches_from_stanford_congress_record.py
Created June 2, 2025 13:40
Code for extracting speeches from the Stanford Congress Record dataset
from copy import copy
from collections import defaultdict, Counter
from functools import cache
from glob import glob
import pandas as pd
import json
from tqdm import tqdm
@cache
@macleginn
macleginn / primer_on_extracting_attention_weights.py
Created April 18, 2025 11:29
An example of extracting attention weights from a transformer encoder in PyTorch that works with PyTorch 2.6 with some explanatory comments.
import torch
import torch.nn as nn
from collections import defaultdict
# --- Hook Implementation ---
# Dictionary to store attention weights during the forward pass
# Structure: {layer_index: attention_weights_tensor}
# The tensor shape will be (batch_size, num_heads, seq_len, seq_len)
@macleginn
macleginn / bpe.py
Created November 15, 2024 10:33
BPE without word separation
import urllib.request
from collections import defaultdict
raw_bytes = urllib.request.urlopen(
'http://www.sls.hawaii.edu/bley-vroman/brown.txt')
brown_corpus = raw_bytes.read().decode('utf8').replace('\r\n', '\n')
B = brown_corpus[:250]
def find_pairs(text):
pairs = defaultdict(int)