Created
February 24, 2026 09:52
-
-
Save JGalego/26d617e5c939af0c32f3c16e4e392803 to your computer and use it in GitHub Desktop.
AttoGPT ⚛️ Quark: Hi, I'm 10⁻¹⁸ meters and I hold the Universe together! / MicroGPT: I'm 50 LoC and I generate baby names. Wanna feel inadequate together?
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """ | |
| ▄▀█ ▀█▀ ▀█▀ █▀█ █▀▀ █▀█ ▀█▀ ⚛️ | |
| █▀█ █ █ █▄█ █▄█ █▀▀ █ | |
| And God said, "Let there be quarks," and the quarks were very small. And God said, "Let there be MicroGPT," and it was even smaller. | |
| """ | |
| import os,math,random as R | |
| if not os.path.exists("input.txt"):import urllib.request;urllib.request.urlretrieve("https://raw.githubusercontent.com/karpathy/makemore/988aa59/names.txt","input.txt") | |
| D=[s.strip()for s in open("input.txt")if s.strip()];R.shuffle(D);C=sorted(set("".join(D)));B=len(C);V=B+1 | |
| class X: | |
| def __init__(s,d,c=(),l=()):s.d,s.g,s.c,s.l=d,0,c,l | |
| def __add__(a,b):b=b if isinstance(b,X)else X(b);return X(a.d+b.d,(a,b),(1,1)) | |
| def __mul__(a,b):b=b if isinstance(b,X)else X(b);return X(a.d*b.d,(a,b),(b.d,a.d)) | |
| def __pow__(s,e):return X(s.d**e,(s,),(e*s.d**(e-1),)) | |
| def __neg__(s):return s*-1 | |
| def __radd__(s,o):return s+o | |
| def __sub__(a,b):return a+-b | |
| def __rmul__(s,o):return s*o | |
| def __truediv__(a,b):return a*b**-1 | |
| log=lambda s:X(math.log(s.d),(s,),(1/s.d,));exp=lambda s:(lambda e:X(e,(s,),(e,)))(math.exp(s.d));relu=lambda s:X(max(0,s.d),(s,),(s.d>0,)) | |
| def bw(s): | |
| t,v=[],set() | |
| def f(x):(v.add(x),[f(c)for c in x.c],t.append(x))if x not in v else 0 | |
| f(s);s.g=1;[setattr(c,'g',c.g+l*n.g)for n in t[::-1]for c,l in zip(n.c,n.l)] | |
| L,E,T,H=1,16,16,4;HD=E//H;M=lambda r,c:[[X(R.gauss(0,.08))for _ in range(c)]for _ in range(r)];Z=X(0) | |
| S={"te":M(V,E),"pe":M(T,E),"lm":M(V,E)};[S.setdefault(f"{i}{k}",M(E,E))for i in range(L)for k in"qkvo"];[S.setdefault(f"{i}{k}",M(E*(4 if k=="1"else 1),E*(1 if k=="1"else 4)))for i in range(L)for k in"12"] | |
| P=[p for m in S.values()for r in m for p in r];li=lambda x,w:[sum((a*b for a,b in zip(r,x)),Z)for r in w] | |
| sm=lambda x:(lambda m,e:(lambda z:[u/z for u in e])(sum(e,Z)))(*((lambda m:[(u-m).exp()for u in x])(max(u.d for u in x)),)*2) | |
| rms=lambda x:(lambda s:[u*(s+1e-5)**-.5 for u in x])(sum((u*u for u in x),Z)/len(x)) | |
| def g(t,p,K,VV): | |
| x=rms([a+b for a,b in zip(S["te"][t],S["pe"][p])]) | |
| for i in range(L): | |
| r=x;y=rms(x);q,k,v=li(y,S[f"{i}q"]),li(y,S[f"{i}k"]),li(y,S[f"{i}v"]);K[i]+=[k];VV[i]+=[v] | |
| a=[j for h in range(H)for j in(lambda s,qh,kh,vh:(lambda w:[sum((w[t]*vh[t][j]for t in range(len(vh))),Z)for j in range(HD)])(sm([sum((qh[j]*t[j]for j in range(HD)),Z)/HD**.5 for t in kh])))(h*HD,q[h*HD:h*HD+HD],[u[h*HD:h*HD+HD]for u in K[i]],[u[h*HD:h*HD+HD]for u in VV[i]])] | |
| x=[u+v for u,v in zip(li(a,S[f"{i}o"]),r)];r=x;x=[u+v for u,v in zip(li([u.relu()for u in li(rms(x),S[f"{i}1"])],S[f"{i}2"]),r)] | |
| return li(x,S["lm"]) | |
| lr,b1,b2,e=.01,.85,.99,1e-8;m1,m2=[0.]*len(P),[0.]*len(P);N=200 | |
| for step in range(N): | |
| t=[B]+[C.index(c)for c in D[step%len(D)]]+[B];n=min(T,len(t)-1) | |
| if n<1:continue | |
| K,VV=[[]for _ in range(L)],[[]for _ in range(L)];loss=sum([-sm(g(t[p],p,K,VV))[t[p+1]].log()for p in range(n)],Z)/n;loss.bw();a=lr*(1-step/N) | |
| for i,p in enumerate(P):g0=p.g;m1[i]=b1*m1[i]+(1-b1)*g0;m2[i]=b2*m2[i]+(1-b2)*g0*g0;p.d-=a*(m1[i]/(1-b1**(step+1)))/((m2[i]/(1-b2**(step+1)))**.5+e);p.g=0 | |
| print(f"{step+1:4d}/{N} {loss.d:.4f}",end="\r") | |
| print("\n--- samples ---") | |
| for _ in range(20): | |
| K,VV,t,o=[[]for _ in range(L)],[[]for _ in range(L)],B,[] | |
| for p in range(T): | |
| t=R.choices(range(V),weights=[u.d for u in sm([z/.5 for z in g(t,p,K,VV)])])[0] | |
| if t==B:break | |
| o+=[C[t]] | |
| print("".join(o)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Sketchware