timm · November 30, 2025 22:26 · timm · Oct 31, 2025
diff --git a/_Litr.md b/_Litr.md
diff --git a/abcd b/abcd
 #!/usr/bin/env gawk -f
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 BEGIN { FS=" " }
      { seen($1,$2) }
 END   { report() }

 function seen(want,got,    kl) {
  if(!(want in cf)) { cf[want]["tn"] = total }
  if(!(got  in cf)) { cf[got]["tn"]  = total }
  total++
  for(kl in cf) {
    if(kl==want) { cf[kl]["tp"]+=(got==want); cf[kl]["fn"]+=(got!=want) }
    else         { cf[kl]["fp"]+=(got==kl);   cf[kl]["tn"]+=(got!=kl) }}}

 function report(      kl,a,b,c,d) {
  print "    N    TN    FN    FP    TP    PD  PREC    PF   ACC LABEL" 
  for(kl in cf) {
    a = cf[kl]["tn"]; b = cf[kl]["fn"]
    c = cf[kl]["fp"]; d = cf[kl]["tp"]
    printf "%5d %5d %5d %5d %5d %5d %5d %5d %5d %-20s\n",
      d+b, a, b, c, d,
      div(d,d+b), div(d,c+d), div(c,c+a), div(d+a,a+b+c+d), kl | "sort -n" }}

 function div(a,b) { return int(100*a/(b+1E-32)) }
diff --git a/abcd.md b/abcd.md
diff --git a/best b/best
 #!/usr/bin/env gawk -f
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 BEGIN { FS=","}
      { gsub(/[ \t\r]/,"") }
 NR==1 { head(); print $0 ",Y-"  }
 NR>1  { body() }
 END   { tail() }

 function head( i) {
  for(i=1;i<=NF;i++)  {
    if($i ~ /[\+\-]$/) { 
      hi[i] = -(lo[i]=1e32)
      y[i]  = $i ~ /\+$/ }}}
   
 function body(    i,v) {
  for(i=1;i<=NF;i++) {
    if(i in y) {
      $i += 0
      if ($i > hi[i]) hi[i] = $i
      if ($i < lo[i]) lo[i] = $i }
    row[NR-1][i] = $i  }}

 function norm(i,x) { return (x - lo[i]) / (hi[i] - lo[i] + 1e-32) }

 function dist(row,    d,n) {
  for(i in y) { d += (norm(i,row[i]) - y[i])^2; n++ }
  return (d / (n+1e-32))^.5 }

 function tail(    r,i,s,sep,com) {
  for(r in row) {
    com = com ? com : "sort -t, -nk"(1+length(row[r]))
    s = sep=""
    for(i=1;i<=NF;i++) {
      s = s sep row[r][i]; sep = "," }
    print s "," dist(row[r]) | com  }}
diff --git a/bins b/bins
 #!/usr/bin/env gawk -f
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 BEGIN { FS=","
        BINS=7; CONVFMT = "%.2f" }
      { gsub(/[ \t\r]/,"") }
 NR==1 { head(); print }
 NR>1  { body() }
 END   { tail() }

 function head(    i,b) {
  for(i=1;i<=NF;i++) 
    if (($i ~ /^[A-Z]/) && ($i !~ /[-+!]$/))  {
      hi[i] = -(lo[i] = 1E32)
      for(b=0; b<=BINS; b++) bmin[i][b] = 1E32 }}

 function body(    i) {
  for(i=1;i<=NF;i++) row[NR-1][i] = seen(i,$i) }

 function tail(    r,i,s,sep) {
  for(r in row) for(i=1;i<=NF;i++) bin(i,row[r][i])
  for(r in row) {
    s=sep=""
    for(i=1;i<=NF;i++) {
      s = s sep bin(i, row[r][i]); sep = "," }
    print s }}

 function seen(i,v,    d) {         
  if ((v!="?") && (i in hi)) {
    v      += 0
    cnt[i] += 1
    d       = v - mu[i]
    mu[i]  += d/cnt[i]
    m2[i]  += d*(v-mu[i]) 
    sd[i]   = cnt[i] < 2 ? 0 : sqrt(m2[i]/(cnt[i]-1)) }
  return v }

 function bin(i,v,    b) {
  if ((v!="?") && (i in hi)) {
    b = int(BINS / (1 + exp(-1.704 * (v - mu[i]) / (sd[i] + 1E-32))))
    if (v < bmin[i][b]) bmin[i][b] = v
    v = bmin[i][b] }
  return v }
diff --git a/bins.lua b/bins.lua
 #!/usr/bin/env lua
 local help = [[
 bins.lua : stochastic incremental XAI
 (c) 2025, Tim Menzies, [email protected], mit-license.org

 Options:
  -h             Show help.
  -b  bins=7     Number of bins for discretization.
  -e  era=30     Update model every `era` number of rows.
  -r  ruleMax=3  Max conditions in a rule.
  -s  seed=42    Random number seed.
  -f  file=../lua6/auto93.csv  ]]

 local function coerce(s) 
  if s then return tonumber(s) or s:match'^%s*(.-)%s*$' end end

 local the={}; for k,v in help:gmatch("(%S+)=(%S+)") do the[k] = coerce(v) end
 math.randomseed(the.seed)

 local DATA, NUM, SYM, COLS, clone, adds
 local abs,exp,sqrt,log = math.abs, math.exp, math.sqrt, math.log
 local max,rand,cos = math.max, math.random, math.cos

 local fmt = string.format
 local sort = function(t,f) table.sort(t,f); return t end 
 local lt = function(f) return function(a,b) return f(a) < f(b) end end
 local cat = function(a) return "{".. table.concat(a," ") .."}" end

 local function o(v,     list,dict)
  list = function(a,     u) 
           for _,v in ipairs(a) do u[1+#u] = o(v) end; return cat(u) end
  dict = function(d,     u)
           for k,v in pairs(d)  do u[1+#u] = fmt(":%s %s", k, o(v)) end 
           return cat(sort(u)) end
  return type(v) == "number" and fmt(v%1==0 and "%.0f" or "%.3f", v) or
         type(v) ~= "table" and tostring(v) or (#v>0 and list or dict)(v,{}) end

 local function s2a(s,   a)
  a={}; for s1 in s:gmatch"([^,]+)" do a[1+#a] = coerce(s1) end; return a end

 local function csv(file,    src)
  src = assert(io.open(file))
  return function(    s)
    s = src:read()
    if s then return s2a(s) else src:close() end end end

 local function cut(a0,n,  data)
  local a1,a2 = {},{}
  for j,v in ipairs(a0) do if j <= n then a1[1+#a1]=v else a2[1+#a2]=v end end
  if data then return clone(data,a1),clone(data,a2) end
  return a1,a2 end

 function box_muller(mu,sd) 
  return mu + sd * sqrt(-2 * log(rand())) * cos(6.28 * rand()) end

 -- ----------------------------------------------------------------------------
 function DATA(  src) return adds(src, {n=0,rows={},cols=nil}) end

 function clone(i,  src) return adds(src, DATA{i.cols.names}) end

 function NUM(at,s) 
  return {at=at or 0, of=s, n=0, mu=0, m2=0, sd=0, bins={},
          best=(tostring(s) or ""):find"+$" and 1 or 0} end

 function SYM(at,s) return {at=at, of=s, n=0, has={}, bins={}} end

 function COLS(row,    t,x,y,all,col)
  x,y,all = {},{},{}
  for n,s in ipairs(row) do
    col = (s:match"^[A-Z]" and NUM or SYM)(n,s)
    all[n] = col
    if not s:match"X$" then
      t = s:find"[+-]$" and y or x
      t[1+#t] = col end end 
  return {all=all, x=x, y=y, names=row} end

 local function add(i,v)
  if v == "?" then return v end
  i.n = i.n + 1
  if i.has then i.has[v] = 1 + (i.has[v] or 0) 
  elseif i.mu then 
    local d = v - i.mu
    i.mu = i.mu + d / i.n
    i.m2 = i.m2 + d * (v - i.mu)
    i.sd = i.n<2 and 0 or sqrt((i.m2/(i.n - 1))) 
  elseif i.rows then  
    if not i.cols then i.cols = COLS(v) else
      for _,col in pairs(i.cols.all) do add(col, v[col.at]) end 
      i.rows[1 + #i.rows] = v end end
  return v end  

 function adds(src, it)
  it = it or NUM()
  if type(src) == "string" 
  then for row in csv(src) do add(it,row) end
  else for _,row in pairs(src or {}) do add(it,row) end end
  return it end

 local function norm(i,v)
  return (i.has or v=="?") and v 
         or 1/(1 + math.exp(-1.7 * (v - i.mu)/(i.sd + 1e-32))) end

 local function disty(i,row,     d)
  d=0; for _,y in pairs(i.cols.y) do d= d + (norm(y, row[y.at]) - y.best)^2 end
  return sqrt(d/#i.cols.y)  end

 local function distys(i,  rows,      y)
   y = function(row) return disty(i, row) end
   return sort(rows or i.rows, function(r1,r2) return y(r1) < y(r2) end) end

 local function two(data) 
  local train,test,start,todo,seen,best,rest,d
  shuffle(data.rows)
  train,test = cut(data.rows, data.n//2)
  start,todo = cut(train, 4)
  seen       = clone(data, start)
  best,rest  = cut(distys(seen),2,data)
  d          = function(row,what) return distx(seen, row, mid(what)) end
  for n,row in pairs(todo) do
    if n>256 then break end
    if d(row,best) < d(row,rest) then 
      add(seen, add(best, row)) 
      if best.n > sqrt(seen.n) then 
        add(rest, sub(best, table.remove( distys(best)))) end end end 
  distys(best)
  return {best=best, rest=rest, seen=seen, test=test,
          model=function(row) return d(row,best) - d(row, rest) end} end

 -- ----------------------------------------------------------------------------
 local egs={}

 egs["-h"] = function(_) print("\n"..help.."\n") end
 egs["-s"] = function(n) math.randomseed(n); the.seed =n end
 egs["--the"] = function(_) print(o(the)) end
 egs["--csv"] = function(_) for row in csv(the.file) do print(o(row)) end end

 egs["--num"] = function(_,num)
  num=NUM()
  for _=1,1000 do add(num, box_muller(10,5)) end
  print(fmt("%.3f %.3f", num.mu, num.sd)) end 

 egs["--data"] = function(_) 
  for n,col in pairs(DATA(the.file).cols.x) do 
  print(n,o(col)) end end

 egs["--disty"]= function(_,    data,num)
  data,t = DATA(the.file), {}
  distys(data)
  for n,row in pairs(data.rows) do t[n]=disty(data,row) end
  print(o(t)) end 

 local function cli(d,funs)
  for i,s in pairs(arg) do
    if funs[s]
    then funs[s](coerce(arg[i+1]))
    else for k,_ in pairs(d) do
          if k:sub(1,1)==s:sub(2) then d[k]=coerce(arg[i+1]) end end end end end

 if arg[0]:find"bins.lua" then cli(the,egs) end
diff --git a/bins.md b/bins.md
diff --git a/ell b/ell
 #!/usr/bin/env bash
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 alias ls="\ls --color"
 alias reload="source '$Here/ell' && echo ✅"
 alias grep='grep --color=auto'
 alias tree='tree -C'

 export BASH_SILENCE_DEPRECATION_WARNING=1
 export PATH="$Here:$PATH"
 export HISTSIZE=10000
 export HISTFILESIZE=20000
 export HISTCONTROL=ignoredups:erasedups

 Here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 branch() { git branch 2>/dev/null | awk '/^\*/ {print $2}'; }
 dirty() { [[ -n $(git status -s 2>/dev/null) ]] && echo "*"; }
 bold=$(tput bold) col0=$(tput sgr0) col1=$(tput setaf 6) col2=$(tput setaf 3)

 PROMPT_COMMAND='PS1="${bold}${col1}$(basename "$(dirname "$PWD")")/$(basename "$PWD")${col0} ${col2}$(branch)$(dirty)${col0} ▶ "'

 vi() {
  nvim --clean \
    --cmd "let g:netrw_banner=0 | let g:netrw_liststyle=3 | let g:netrw_browse_split=4 | let g:netrw_winsize=15" \
    --cmd "set number relativenumber cursorline mouse=a clipboard=unnamedplus ignorecase smartcase" \
    --cmd "set statusline=%#StatusLine#\ ▶\ %f\ %m%r%=%y\ ❖\ %l:%c\ ❖\ %p%%\ " \
    --cmd "set expandtab tabstop=2 shiftwidth=2 splitright splitbelow" \
    --cmd "set undofile undodir=~/.vim/undo" \
    --cmd "nnoremap Q :quitall<CR>" \
    --cmd "colorscheme zaibatsu" \
    --cmd "set laststatus=2" \
    "$@"
 }

 hi() { 
  clear
  echo "${col1}"
  cat<<'EOF'
 ██╗      ██╗ ████████╗ ██████╗ 
 ██║      ██║ ╚══██╔══╝ ██╔══██╗
 ██║      ██║    ██║    ██████╔╝
 ██║      ██║    ██║    ██╔══██╗
 ███████╗ ██║    ██║    ██║  ██║
 ╚══════╝ ╚═╝    ╚═╝    ╚═╝  ╚═╝
  (cause simple ain't stupid)
 EOF
  echo "${col0}"
 }

 inst() {
  local m=""
  for p in $1; do command -v "$p" &>/dev/null || m+="$p "; done
  [ "$m" ] && case "$(uname -s)" in
    Darwin*) brew install $m ;;
    Linux*)  sudo apt install -y $m ;;
    MINGW*)  winget install $m ;;
  esac
 }

 # only run slow or verbose command at initial startup, not on reload
 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
  inst git nvim gawk tree
  hi
  exec bash --init-file "${BASH_SOURCE[0]}" -i
 fi
diff --git a/eras b/eras
 #!/usr/bin/env gawk -f
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 BEGIN { SEED = 1234567891
        ERA  = 100 }
 NR==1 { srand(SEED)
        print "\n" $0; next }
      { a[rand()] = $0 
        if (length(a) > ERA) dump(a) }
 END   { dump(a) }

 function dump(a,    i) {
  if (length(a) > 0) {
    print ""
    for(i in a) print a[i]
    delete a }}
diff --git a/head b/head
 #!/usr/bin/env sh
 figlet -W -f mini $@ | gawk '{print "--    " $0}'
diff --git a/lua.ssh b/lua.ssh
 # Lua.ssh --- Sheet definitions for Lua source code
 # Copyright (c) 2014 Kenji Rikitake
 # Copyright (c) 1999 Edward Arthur, Akim Demaille, Miguel Santana
 #

 #
 # This file is NOT a part of a2ps.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 2, or (at your option)
 # any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; see the file COPYING.  If not, write to
 # the Free Software Foundation, 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.

 ## This style is derived from Edward Arthur's AWK Style Sheet

 style Lua is

 written by "Kenji Rikitake <[email protected]>"
 version is 0.1
 requires a2ps version 4.9.7

 documentation is
    "This style file is intended to support the Lua programming language source code."
 end documentation

 alphabets are
   "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0"
 case sensitive

 keywords in Keyword are
    and, false, ipairs, nil, not, or, pairs, rawequal, rawget, 
    rawlen, rawset, select, tonumber, tostring, true, type
 end keywords

 #keywords in Keyword_strong are
 keywords in Keyword_strong are
    assert, break, collectgarbage, do, dofile, else, elseif, "end", 
    error, for, function, getmetatable, goto, if, "in", local, 
    load, loadfile, next, pcall, print, repeat, require, return, 
    setmetatable, then, until, while, xpcall, _G, _VERSION
 end keywords

 keywords in Label_strong are
    "^" function
 end keywords

 keywords in Comment are
    self
 end keywords

 sequences are
    "[[" Comment "]]",
    "--" Comment,
    C-string
 end sequences
 end style

diff --git a/makefile b/makefile
 # vim: ts=2 sw=2 noet
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 #------------------------------------------------------
 K    ?= 1#
 M    ?= 2#
 BINS ?= 4#
 ERA  ?= 100#
 DATA  = ~/gits/moot/classify/diabetes.csv#
 DATAG = ~/gits/moot/optimize/misc/auto93.csv#

 G	    = gawk -f#
 OK    = gawk -f ok.awk -f#

 #------------------------------------------------------
 SHELL=/bin/bash
 export PATH := $(CURDIR):$(PATH)
 .SILENT:
 .PHONY: help egs ok eg-nbc eg-abcd eg-soybean eg-globals pull push

 help: ## show this help
 	@gawk 'BEGIN { FS=":.*?## ";c="\033[1;3"; r="\033[0m";            \
 		             printf "\n%s6mmake%s [%s3moptions%s]:\n\n",c,r,c,r} \
         NF==2 && $$1~/^[a-z0-9A-Z_-]+/{                              \
 				         printf "  %s2m%-15s%s %s\n",c,$$1,r,$$2}' $(MAKEFILE_LIST)

 #------------------------------------------------------
 hi=@echo -e "\n--------- $@"

 test: eg-nbc eg-abcd eg-soybean eg-globals ## run all egs: now

 ok:; chmod +x nbc abcd bins

 eg-bins: ok $(DATA) ## run naive bayes classifier
 	cat $(DATA) | bins BINS=$(BINS)

 eg-nbc: ok $(DATA) ## run naive bayes classifier
 	$(hi); cat $(DATA) | bins BINS=$(BINS) | nbc K=$K M=$M | sort | uniq -c

 eg-abcd: ok $(DATA) ## run classifier with confusion matrix
 	$(hi); cat $(DATA) | bins BINS=$(BINS) \
 										 | nbc K=$K M=$M #| abcd

 eg-soybean:  ## run classifier on soybean
 	$(hi); $(MAKE) eg-abcd DATA=~/gits/timm/moot/classify/soybean.csv

 eg-best: ok $(DATA) ## run naive bayes classifier
 	cat $(DATAG) | bins BINS=$(BINS) | best | column -s, -t | sed -n '1,5p;395,$$p'

 eg-tree: ok $(DATA) ## run naive bayes classifier
 	cat $(DATAG) | bins BINS=$(BINS) | best | tree

 eg-globals: ok $(DATA) ## run classifier with confusion matrix
 	$(hi); ( cat $(DATA) | $(OK) bins BINS=$(BINS) \
                       | $(OK) nbc K=$K M=$M | $(OK) abcd ) > /dev/null

 # misc ------------------------------------------------
 sh: ## run a customized shell
 	sh ell

 pull: ## update from main
 	git pull

 push: ## commit to main
 	git commit -am saving;  git push; git status

 ~/tmp/%.pdf: %.lua
 	echo "pdf-ing $@ ... "
 	a2ps                          \
 		--file-align=virtual         \
 		--line-numbers=1              \
 		--pro=color                    \
 		--lines-per-page=120 \
 		--pretty=lua.ssh \
 		--left-title=""                 \
 		--borders=no                     \
 	  --right-footer="page %s. of %s#"  \
 		--landscape                        \
 		--columns 3                          \
 		-M letter                             \
 		-o - $^ | ps2pdf - $@
 	open $@
diff --git a/nbc b/nbc
 #!/usr/bin/env gawk -f
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 BEGIN { FS=","; WAIT=20; M=2; K=1 }
      { gsub(/[ \t\r]/,"") }
 NR==1 { head() }
 NR>1  { if (NR > WAIT+1) print $klass, predict()
        train($klass) }

 function head(    i) {
  for(i=1;i<=NF;i++) if ($i ~ /[!]$/) klass = i }

 function train(actual,    i) {
  nk[actual]++
  for(i=1;i<=NF;i++)
    if (i != klass) freq[i][$i][actual]++ }

 function predict(    i,k,like,best,max) {
  max = -1E32
  for(k in nk) {
    like = log((nk[k] + K) / (NR-1 + K*length(nk)))
    for(i=1;i<=NF;i++) 
      if (i != klass) 
        like += log((freq[i][$i][k] + M/length(freq[i])) / (nk[k] + M));
    if (like > max) {
      max = like
      best = k }}
  return best ? best : k }
diff --git a/nbc.md b/nbc.md
diff --git a/ok.awk b/ok.awk
 # Copyright (c) 2025 Tim Menzies, MIT License
 # https://opensource.org/licenses/MIT
 END { rogues() }

 function rogues(    i,s,known) {
  known = "^(NF|NR|FS|RS|RT|FNR|OFS|ORS|" \
             "PREC|ARGC|ARGV|OFMT|LINT|FPAT|" \
             "ERRNO|RSTART|ARGIND|" \
             "SUBSEP|CONVFMT|ENVIRON|SYMTAB|FUNCTAB|PROCINFO|" \
             "FILENAME|RLENGTH|BINMODE|" \
             "IGNORECASE|FIELDWIDTHS|ROUNDMODE|TEXTDOMAIN)$"
  for(i in SYMTAB) if (i !~ known) s = s " " i
  if (s) print "?" s  > "/dev/stderr" }
diff --git a/rules.md b/rules.md
diff --git a/six.lua b/six.lua
 #!/usr/bin/env lua
 local the,l,egs = {},{},{} -- config, lib, demos
 local help = [[
 six : stochastic incremental XAI
 (c) 2025, Tim Menzies, [email protected], mit-license.org

 Options:
  -h             Show help.
  -b  bins=7     Number of bins for discretization.
  -e  era=30     Update model every `era` number of rows.
  -r  ruleMax=3  Max conditions in a rule.
  -s  seed=42    Random number seed.
  -f  file=../lua6/auto93.csv  ]]

 -- ----------------------------------------------------------------------------
 local DATA, NUM, SYM, COLS = {},{},{},{}

 function DATA.new(  rows,     i) 
  i = l.new(DATA,{rows={}, cols=nil})
  for _,row in pairs(rows or {}) do i:add(row) end 
  return i end

 function DATA.clone(i,  rows,     clone)
  clone = DATA:new({i.cols.names})
  for _,row in pairs(rows or {}) do clone:add(row) end 
  return clone end

 function NUM.new(at,s) 
  return l.new(NUM, {at=at or 0, of=s, n=0, mu=0, m2=0, sd=0, bins={}, 
                     best=(tostring(s) or ""):find"+$" and 1 or 0}) end

 function SYM.new(at,s) 
  return l.new(SYM, {at=at, of=s, n=0, bins={}, has={}, mode=0, most=-1}) end

 function COLS.new(row,    x,y,all,col)
  x,y,all = {},{},{}
  for n,s in ipairs(row) do
    col = (s:match"^[A-Z]" and Num or Sym).new(n,s)
    all[n] = col
    if not s:match"X$" then
      l.push(s:find"[+-]$" and y or x, col) end end 
  return l.new(COLS, {all=all, x=x, y=y, names=row}) end

 -- -----------------------------------------------------------------------------
 function DATA.add(i,row) 
  if not i.cols then i.cols = COLS.new(row) else
    for _,col in pairs(i.cols.all) do col:add(row[col.at]) end 
    l.push(i.rows, row) end 
  return row end 

 function SYM.add(i,v)
  if v~="?" then
    i.n = i.n + 1
    i.has[v] = 1 + (i.has[v] or 0)
    if i.has[v] > i.most then i.most,i.mode = i.has[v],v end  end
  return v end

 function NUM.add(i,n,    d)
  if v~="?" then
    i.n  = i.n + 1
    d    = n - i.mu
    i.mu = i.mu + d / i.n
    i.m2 = i.m2 + d * (n - i.mu)
    i.sd = i.n<2 and 0 or (i.m2/(i.n - 1))^0.5 end 
  return n end

 -- -----------------------------------------------------------------------------
 function NUM.norm(i,v)
  return 1/(1 + math.exp(-1.7 * (v - i.mu)/(i.sd + 1e-32))) end

 function SYM.bin(_,v) return v end
 function NUM.bin(i,v) return v=="?" and v or math.floor(the.bins*i:norm(v)) end

 function DATA.disty(i,row,    d)
  d=0; for _,y in pairs(i.cols.y) do d= d + (y:norm(row[y.at]) - y.best)^2 end
  return (d/#(i.cols.y)) ^ 0.5 end

 function DATA.bins(i,row,    v,y)
  y = i:disty(row)
  for _,x in pairs(i.cols.x) do
     v = x:bin(row[x.at]) 
     if v ~= "?" then
       x.bins[v] = x.bins[v] or Num.new(x.at,v)
       x.bins[v]:add(y) end end end

 -- -----------------------------------------------------------------------------
 function DATA.rule(i,stop,    t,u,f,stop)
  f = function(n) return math.floor(100*(n.mu + n.sd/n.n^.5)) end 
  t,u = {},{}
  for _,x in pairs(i.cols.x) do
    for _,col in pairs(x.bins) do
      push(t, col) end end
  for j,col in pairs(sort(t, lt(f))) do
    if stop and j > stop then break end
    u[col.at] = u[col.at] or {}
    push(u[col.at], col) end  -- tests on each attribute are disjunctions
  return u end

 function DATA.or(i,ors,row)
  for _, col in pairs(ors) do 
    if col.of == i.cols.all[col.at]:bin(row[col.at]) then return true end end
  return false end
   
 function DATA.and(i,ands,row)
  for _,ors in pairs(ands) do if not i:or(ors,row) then return false end end
  return true end

 function DATA.selects(i,rule,rows,     u)
  u={}; for _,r in pairs(rows) do if i:and(i,rule,r) then l.push(u,r) end end
  return u end

 function main(file,    data1,y,train,test,rule)
  data1 = Data.new()
  for _,row in csv(file) do data1:add(row) end
  train, test = {},{}
  for j,row in pairs(shuffle(data1.rows)) do
    push(j < #data.rows/2 and train or test, row) end
  rule,data2 = nil, data1:clone()
  y = function(row) return data2:disty(row) end
  for r,row in pairs(train) do
    data2:bins( data2:add(row))
    if r % the.era == 0 then
      rule = data2.rules( math.min(3, #data2.rows/the.era) ) 
      print(Y(sort(data2:selects(rule,test),Y)[1])) end end end

 -- -----------------------------------------------------------------------------
 -- Misc tricks
 function l.new(meta,t) meta.__index=meta; return setmetatable(t,meta) end

 l.fmt = string.format

 function l.push(t,v) t[1+#t]=v; return v end
 function l.sort(t,f) table.sort(t,f); return t end
 function l.lt(f) return function(a,b) return f(a) < f(b) end end

 function l.shuffle(t,    n)
 	for m= #t,2,-1 do n=math.random(m); t[m],t[n] = t[n],t[m] end
  return t end

 -- Thing to string
 local function _a2a(a,u) for _,v in ipairs(a) do push(u,cat(v)) end;return u end

 local function _d2a(d,u)
  for k,v in pairs(d) do 
    if type(v)~=type(_d2a) then
      l.push(u,l.fmt(":%s %s",k,type(v)==type(_d2a) and "" or l.cat(v))) end end
  return l.sort(u) end

 function l.cat(t)
  return type(t) == "number" and l.fmt(t%1==0 and "%s" or "%.3f", t) 
      or type(t) ~= "table"  and tostring(t) 
      or "{".. table.concat((#t>0 and _a2a or _d2a)(t,{})," ") .."}" end

 function l.pat(t) print(l.cat(t)); return t end

 -- String to thing
 function l.coerce(s) 
  if s==nil then return nil end
  return tonumber(s) or s:match'^%s*(.-)%s*$' end

 local function _cells(s,   a)
  a={}; for s1 in s:gmatch"([^,]+)" do l.push(a, l.coerce(s1)) end; return a end

 function l.csv(file,     n,stream)
  n,stream = 0,assert(io.open(file))
  return function(    s)
    s= stream:read()
    if s then n=n+1; return n,_cells(s) else stream:close() end end end

 -- Config management.
 function l.settings(s,     d)
  d={}; for k,v in s:gmatch("(%S+)=(%S+)") do d[k]=l.coerce(v) end; return d end

 function l.cli(d,funs)
  for i,s in pairs(arg) do
    if funs[s]
    then funs[s](l.coerce(arg[i+1]))
    else for k,_ in pairs(d) do
           if k:sub(1,1)==s:sub(2) then d[k]=l.coerce(arg[i+1]) end end end end
  return d end

 -- -----------------------------------------------------------------------------
 egs["-h"] = function(_) print("\n"..help.."\n") end
 egs["-s"] = function(n) math.randomseed(n); the.seed =n end
 egs["--the"] = function(_) l.pat(the) end
 egs["--guess"] = function(_) main(the.file) end

 the = l.settings(help)
 math.randomseed(the.seed)
 if arg[0]:find"six.lua" then the=l.cli(the,egs) end 

 return {DATA=DATA, NUM=NUM, SYM=SYM, COLS=COLS, help=help, the=the, l=l}
diff --git a/tree b/tree
 #!/usr/bin/env lua
 local the = {leaf=2}
 local push, coerce, cells, csv, inc2, cli
 local tree, show, read,_tally, _split, _div, _kids

 function tree(goal,klass,rows,  col,val,op)
  local col1, val1, mu = _split(goal,klass, rows)
  return {rows=rows, col=col, val=val, op=op, mu= mu,
          kids=_kids(goal,klass,rows,col1,val1)} end

 function _split(goal,klass, rows,      all,good,best,col,val,n,p,s,mu)
  all, good, mu = _tally(goal,klass, rows,{},{})
  best = -1
  for c, vals in pairs(good) do
    for v, g in pairs(vals) do
      n = all[c][v]  
      p = g/n
      s = p*p / (p + (n-g)/n + 1e-32) 
      if s > best then best, col, val = s, c, v end end end
  return col, val, mu end

 function _tally(goal,klass,rows,all,good,    y)
  y = 0
  for _, row in pairs(rows) do
    y = y + row[klass]
    for c, v in pairs(row) do
      if c ~= klass and v ~= "?" then
        inc2(all, c, v)
        if row[klass] <= goal then inc2(good, c, v) end end end end
  return all, good, y/#rows end

 function _kids(goal,klass,rows,col1,val1,      out)
  out = {}
  for op, t in pairs(_div(col1, val1, rows)) do
    if #t >= the.leaf and #t < #rows then
      out[op] = tree(goal,klass, t, col1, val1, op) end end 
  return out end

 function _div(c,v,rows,     yes,no)
  yes, no = {}, {}
  for _, r in pairs(rows) do push((r[c]=="?" or r[c]==v) and yes or no, r) end
  return {["=="] = yes, ["!="] = no} end

 ----------------------------------------------------------------
 function show(t,  lvl)
  print(string.format("%s%s (n=%d)", ("|.. "):rep(lvl or 0),
        t.col and (t.col.." "..t.op.." "..t.val) or "", #t.rows))
  for _, kid in pairs(t.kids or {}) do
    show(kid, (lvl or 0) + 1) end end

 function read(  rows,klass)
  rows = {}
  for row in csv() do
    if not klass then
      print(row)
      for i, s in pairs(row) do if s:find"-$" then klass = i end end
    else rows[#rows + 1] = row end end
  table.sort(rows, function(t,u) return t[klass] < u[klass] end)
  return rows[math.floor((#rows)^.5)][klass], klass, rows end

 -------------------------------------------------------------------------------
 function push(t,x)    t[#t + 1] = x; return x end
 function coerce(s)    return tonumber(s) or s end
 function cells(s,  t)
  t = {}; for s1 in s:gmatch"([^,]+)" do t[#t + 1] = coerce(s1) end
  return t end

 function csv()
  return function() local s = io.read(); return s and cells(s) end end

 function inc2(t,c,v) 
  t[c]=t[c] or {}; t[c][v]=(t[c][v] or 0)+1; return t[c][v] end

 function cli()
  for i, s in pairs(arg or {}) do
    if the[s:sub(2)] then the[s:sub(2)] = coerce(arg[i+1]) end end end

 -------------------------------------------------------------------------------
 if ... == nil then
  cli()
  local goal,klass,rows = read()
  show(tree(goal, klass, rows)) end

 return {tree=tree, show=show, read=read} 



 """

 def mad(rows):
  a = [y(row) for row in rows]
  median = sorted(a)[len(a) // 2]
  return sorted([abs(x - median) for x in a])[len(a) // 2]

 def mads(a,b): n1,n2=len(a),len(b); return (n1*mad(a) + n2*mad(b))/(n1+n2)

 def key(rows,col,klass): 
  x = lambda row:row[col]  
  y = lambda row:row[klass]
  rows = sorted([row in rows if x(row) != "?"]), key=x)
  most = mad(rows)
  b4,min = [],[],-1,len(rows)**.5
  for i in range(len(rows)):
    b = int(BINS/(1+e^(-1.704*(mu - x(rows[i])/sd)) 
    left += [row]
    if b != b4 and i>min and len(rows) - i > min: 
      mad1,mad2 = mads(left),mads(rows[i:])
      if now < most
        = 
    b4 = b

 for row in sorted(rows, key=lambda row

 function span(t,i, j,    u)
  u,j = {},j or #t; for k=i,j do u[1+#u] = t[k] end; return u end

 function cut(rows,col,klass)
  x  = function(r) return r[col] end
  y  = function(r) return r[klass] end
  lt = function(r1,r2) return x(r1) < x(r2) end
  t  = {}; for _,r in pairs(rows) if x(r) != "?" then t[1+#t]=r end end
  table.sort(t,lt)
  most = sd(t,y)
  for i=1,#t do
    j=#t-i
    b = (the.BINS/(1+math.exp(-1.7404*math.exp((mu- x(t[i]))/sd))))//2
    left[1 + #left] = rows[i]
    if b ~= b4 and i>min and j > min:
      sd1,sd2 = sd(span(rows,1,i),y), sd(span(rows,i+1))) / (i+j)
      now = (i * sd1 + j * sd2) / (i+j)
      if now < least then
        out={now, {"<=",col,x(rows[i])}, {">",col,x(rows[i+1])}

      

 function sd(row,fn,     v,d,n,mu,m2)
  n,mu,m2 = 0,0,0
  for i=1,#t do
    v = fn(t[i])
    if v~="?" then
      n,d = n+1, v-mu
      mu = mu+d/n; m2 = m2+d*(x-mu) end end
  return n<2 and 0 or (m2/(n-1))^.5 or 0 end
diff --git a/tree.py b/tree.py
 import fileinput,random,sys,re
 from math import exp,min
 from bisect import insort, bisect_left

 class o(dict): __getattr__=dict.__getitem__; __setattr__=dict.__setitem__

 the = o(BINS=10,LEAF=20,FILE=None,HEIGHT=3)

 # misc details
 def is_num(s)  : return s[0].isupper()
 def is_x(s)    : return s[-1] not in "+-!X"
 def is_goal(s) : return {"+":1,"-":0}.get(s[-1]) 

 def selects(split, row): 
  if (z := row[split.col]) == "?": return True
  if split.op == "<" : return z <  split.val
  if split.op == "<=": return z <= split.val
  if split.op == "==": return z == split.val
  if split.op == ">=": return z >= split.val
  if split.op == ">" : return z >  split.val

 ## Types are UPPERCASE(). Used only in constructor.
 def SPLIT(op, col, val):          return o(op=op, col=col, val=val)
 def DATA(rows, y, names):         return o(rows=rows, y=y, names=names)
 def TREE(rows, split, mid, kids): return o(rows=rows, split=split, mid=mid, kids=kids)
 def STATS(n, mid, div):           return o(n=n, mid=mid, div=div)

 # Constructors have Leadingcapitals() and return TYPES
 Split = SPLIT # simple constructors are types

 def Stats(rows, i):
  return STATS(*_stats(sorted(r[i] for r in rows if r[i]!="?")))

 def _stats(s):
  n = len(s)
  if n>= 4: mid,div = s[n//2],          s[3*n//4] - s[n//4]
  else:     mid,div = (s[-1] - s[0])/2, s[-1] - s[0]
  return n, mid, div / 1.349

 def Data(things):
  names, ys, rows, lo, hi = None, [], {}, {}, {}
  for i, row in enumerate(things):
    if i == 0: 
      names = row
      ys = {i:w for i,name in enumerate(names) if (w:=is_goal(name))}
    else: 
      rows += [row]
      for j, name in enumerate(names):
        if is_num(name) and row[j] != "?":
          lo[j] = min(lo.get(j, 1e32), row[j])
          hi[j] = max(hi.get(j, -1e32), row[j])
  def norm(i,n): return (n - lo(i)) / (hi[i] - lo[i] + 1e-32)
  def Y(i,n)   : return sqrt(sum((norm(i,n) - w)**2 for i,w in ys.items()))
  return DATA(rows, Y, names)

 def Tree(data, split=None, lvl=0):
  return TREE(data.rows, split,
              Stats(data.rows, data.y).mid,
              list(_kids(data, _splits(data), lvl)))

 #-----------------------------------------------------------------------------
 def _kids(data, splts, lvl):
  if splts and lvl < the.HEIGHT:
    for split in splts:
      subset = [row for row in data.rows if selects(split, row)]
      if the.LEAF <= len(subset) < len(data.rows):
        yield Tree(Data(subset, data.y, data.names), split, lvl+1)

 def _splits(data):
  least, out = 1e32, []
  for col, name in enumerate(data.names):
    if is_x(name):
      if xys := sorted((r[col],data.y(r)) for r in data.rows if r[col] != "?")
        tmp,splts = (_numSplits if is_num(name) else _symSplits)(xys, col)
        if splts and tmp < least:  
          least,out = tmp,splts
  return out

 def _symSplits(xys,col):
  d = {}
  for x,y in xys:
    d[x]  = d.get(x, [])
    d[x] += [y]
  return sum(len(a)/len(rows) * _stats(sorted(a)).div [-1] for a in d.values()), \
         [Split("==", col, v) for v in d]

 #-for get insort etc. get all theys,reviersed. appedl, push. then only resort wwn x~=b4
 def _numSplits(xys,col):
  n, b4, best, cuts = len(rows), xys[0][0], 1e32, []
  lo, hi = [], sorted(y for _,y in xys)
  for i, (x,y) in enumerate(xys):
    insort(lo, hi.pop( bisect_left(hi,y))) # move y hi ==> lo
    if x != b4:
      if best > (xpect := (i*_stats(lo)[-1] + (n-i)*_stats(hi)[-1]) / n):
        best, cuts = xpect, [Split("<", col, x), Split(">=", col, x)]
    b4 = x
  return best, cuts

 def show(tree, lvl=0):
  t = tree
  cond = f"{t.split.col} {t.split.op} {t.split.val} " if t.split else ""
  print(f"{'|.. ' * lvl}{cond} (w= {t.mid}, n={len(t.rows)})")
  [show(kid, lvl+1) for kid in t.kids]

 def leaf(tree, row):
  for k in tree.kids:
    if selects(k.split, row): return leaf(k, row)
  return tree

 #------------------------------------------------------------------------------
 def coerce(x):
  for f in (int,float):
    try: return f(x)
    except: pass
  return x

 def csv(path):
  with fileinput.input(files=path or (), encoding="utf-8") as f:
    for s in f:
      if (s := re.sub(r"\s+|#.*", "", s)):
        yield [coerce(x) for x in re.split(",", s)]

 def _discretize(col, rows, i):
  s = Stats(rows, i)
  lo, bin = {}, lambda z: int(the.BINS/(1 + exp(-1.7 * (z - s.mid)/s.div))) 
  for n in col:
    if n != "?":
      lo[bin(n)] = min(lo.get(bin(n), 1e32), n)
  return [lo[bin(n)] if n != "?" else "?" for n in col]

 def discretized(data):
  new = []
  for i, (name, col) in enumerate(zip(data.names, zip(*data.rows))):
    if is_num(name):
      col = _discretize(col, data.rows, i)
    new += [col]
  return DATA(list(zip(*new)), data.y, data.names)
  
 #--------------------------------------------------------------------
 if __name__ == "__main__":
  if len(sys.argv) > 1:
    the.FILE = sys.argv[1]
    data = read(csv(the.FILE))
    disc = discretized(data)
    for row in disc.rows[:10]:
      print(row)
diff --git a/two.1 b/two.1
 .\" Manpage for two.lua
 .\" Contact [email protected]
 .TH TWO.LUA 1 "November 29, 2025" "1.0" "User Commands"
 .SH NAME
 two.lua \- stochastic incremental Explainable AI (XAI) tool
 .SH SYNOPSIS
 .B ./two.lua
 .RI [ OPTIONS ]
 .RI [ COMMANDS ]
 .SH DESCRIPTION
 .B two.lua
 is a Lua script that implements a stochastic incremental Explainable AI (XAI) algorithm. It processes data from CSV files to cluster data into "best" and "rest" sets, calculates distances between rows using weighted Euclidean distance (for attributes) or normalized distance to goals (for class variables), and incrementally updates a model.
 .PP
 The script handles both numeric and symbolic data types, supports normalization, and includes a test suite for demonstrating various internal statistical functions.
 .SH OPTIONS
 These options configure the hyperparameters and inputs for the algorithm. They must be provided as key-value pairs or flags.
 .TP
 .B \-h
 Show the help message and exit.
 .TP
 .BI \-b " bins"
 Set the number of bins for discretization.
 .IP
 Default: 7
 .TP
 .BI \-e " era"
 Set the frequency (number of rows) at which the model updates.
 .IP
 Default: 30
 .TP
 .BI \-f " file"
 Specify the path to the input CSV file. The file should contain a header row. Columns starting with uppercase letters are treated as Numerics, others as Symbols. Columns ending in '+' or '-' are treated as optimization goals (maximize/minimize). Columns ending in 'X' are ignored.
 .IP
 Default: ../lua6/auto93.csv
 .TP
 .BI \-r " ruleMax"
 Set the maximum number of conditions allowed in a rule.
 .IP
 Default: 3
 .TP
 .BI \-s " seed"
 Set the random number generator seed for reproducibility.
 .IP
 Default: 42
 .SH COMMANDS
 The script includes several built-in test hooks and execution modes. These arguments trigger specific functions within the script.
 .TP
 .B --csv
 Read the input file defined by \fB-f\fR and print every parsed row to standard output.
 .TP
 .B --cut
 Demonstrate the list splitting (cutting) function.
 .TP
 .B --data
 Load the data file and display summary statistics for the independent variables (X columns).
 .TP
 .B --distx
 Calculate and display distance statistics between rows based on independent variables (X).
 .TP
 .B --disty
 Calculate and display distance statistics based on dependent optimization goals (Y).
 .TP
 .B --inc
 Demonstrate incremental data updates (adding and removing rows from the statistical model).
 .TP
 .B --mode
 Demonstrate the mode (most frequent item) calculation.
 .TP
 .B --num
 Demonstrate Gaussian random number generation and summary statistics (mean, standard deviation).
 .TP
 .B --shuffle
 Demonstrate the array shuffling function.
 .TP
 .B --the
 Print the current configuration settings (the `the` table).
 .TP
 .B --two
 Run the main stochastic incremental XAI clustering algorithm. This splits the data into training/testing sets, performs clustering, and outputs model performance metrics.
 .SH EXAMPLES
 .B Run the main XAI algorithm with default settings:
 .PP
 .nf
    ./two.lua --two
 .fi
 .PP
 .B Process a custom dataset with a specific random seed:
 .PP
 .nf
    ./two.lua -f data/weather.csv -s 101 --two
 .fi
 .PP
 .B Inspect the columns of a dataset:
 .PP
 .nf
    ./two.lua -f ../lua6/auto93.csv --data
 .fi
 .SH EXIT STATUS
 Returns 0 on successful execution.
 .SH AUTHOR
 Tim Menzies ([email protected])
 .SH LICENSE
 MIT License (mit-license.org)
 .SH COPYRIGHT
 Copyright (c) 2025 Tim Menzies.
diff --git a/two.lua b/two.lua
 #!/usr/bin/env lua
 --      __                           
 --     /\ \__                        
 --     \ \ ,_\   __  __  __    ___   
 --      \ \ \/  /\ \/\ \/\ \  / __`\ 
 --       \ \ \_ \ \ \_/ \_/ \/\ \L\ \
 --        \ \__\ \ \___x___/'\ \____/
 --         \/__/  \/__//__/   \/___/ 

 local help = [[
 two.lua : stochastic incremental XAI
 (c) 2025, Tim Menzies, [email protected], mit-license.org

 Options:
  -h             Show help.
  -b  bins=7     Number of bins for discretization.
  -e  era=30     Update model every `era` number of rows.
  -r  ruleMax=3  Max conditions in a rule.
  -s  seed=42    Random number seed.
  -f  file=../lua6/auto93.csv  ]]

 -- coerce(s) --> v ;; Return int or float or bool or string from `s`.
 local function coerce(s) 
  if s then return tonumber(s) or s:match'^%s*(.-)%s*$' end end

 local the={}; for k,v in help:gmatch("(%S+)=(%S+)") do the[k] = coerce(v) end
 math.randomseed(the.seed)

 local DATA, NUM, SYM, COLS, clone, adds

 --     |  o  |_  
 --     |  |  |_) 

 local abs,exp,sqrt,log = math.abs, math.exp, math.sqrt, math.log
 local max,rand,cos = math.max, math.random, math.cos

 local say=io.write
 local fmt = string.format

 -- sort(t,f) --> t ;; Sort `t` using function `f`.
 local sort = function(t,f) table.sort(t,f); return t end 
 -- lt(f) --> f ;; Return a function that sorts `a` and `b` on `f`.
 local lt = function(f) return function(a,b) return f(a) < f(b) end end
 -- cat(a) --> s ;; Return a string representation of array `a`.
 local cat = function(a) return "{".. table.concat(a," ") .."}" end

 -- o(v) --> s ;; Return a string representation of `v`.
 local function o(v,     list,dict)
  list = function(a,     u) 
           for _,v in ipairs(a) do u[1+#u] = o(v) end; return cat(u) end
  dict = function(d,     u)
           for k,v in pairs(d)  do u[1+#u] = fmt(":%s %s", k, o(v)) end 
           return cat(sort(u)) end
  return type(v) == "number" and fmt(v%1==0 and "%.0f" or "%.3f", v) or
         type(v) ~= "table" and tostring(v) or (#v>0 and list or dict)(v,{}) end

 -- s2a(s) --> a ;; Return array of words from string `s`, split on ",".
 local function s2a(s,   a)
  a={}; for s1 in s:gmatch"([^,]+)" do a[1+#a] = coerce(s1) end; return a end

 -- csv(file) --> f ;; Iterator that returns rows from `file`.
 local function csv(file,    src)
  src = assert(io.open(file))
  return function(    s)
    s = src:read(); if s then return s2a(s) else src:close() end end end

 -- shuffle(t) --> t ;; Randomly shuffle the order of elements in `t`.
 local shuffle = function(t,    n)
 	for m=#t,2,-1 do n=math.random(m); t[m],t[n]=t[n],t[m] end; return t end

 -- cut(a0,n,data) --> t,t ;;Split `a0` at `n` (if `data` exists,split that too).
 local function cut(a0,n,  data)
  local a1,a2 = {},{}
  for j,v in ipairs(a0) do if j <= n then a1[1+#a1]=v else a2[1+#a2]=v end end
  return data and clone(data,a1),clone(data,a2) or a1,a2 end

 -- mode(d) --> v ;; Return the most frequent key in `d`.
 local function mode(d,   v,n)
  v,n = nil,0
  for v1,n1 in pairs(d) do if n1>n then v,n=v1,n1 end end
  return v end 

 -- box_muller(mu,sd) --> n ;; Return a random number from a Gaussian `mu`,`sd`.
 function box_muller(mu,sd) 
  return mu + sd * sqrt(-2 * log(rand())) * cos(6.28 * rand()) end

 --      _  |   _.   _   _   _    _ 
 --     (_  |  (_|  _>  _>  (/_  _> 

 -- DATA(src) --> DATA ;; Create a new DATA, populated with `src`.
 function DATA(  src) return adds(src, {n=0,rows={},cols=nil}) end

 -- clone(i,src) --> DATA ;; Return a new DATA with same structure as `i`.
 function clone(i,  src) return adds(src, DATA{i.cols.names}) end

 -- NUM(at,s) --> NUM ;; Create a NUM object to summarize numbers.
 function NUM(at,s) 
  return {at=at or 0, of=s, n=0, mu=0, m2=0, sd=0,
          best=(tostring(s) or ""):find"+$" and 1 or 0} end

 -- SYM(at,s) --> SYM ;; Create a SYM object to summarize symbols.
 function SYM(at,s) return {at=at, of=s, n=0, has={}} end

 -- COLS(row) --> COLS ;; Create a COLS object from a list of column names.
 function COLS(row,    t,x,y,all,col)
  x,y,all = {},{},{}
  for n,s in ipairs(row) do
    col = (s:match"^[A-Z]" and NUM or SYM)(n,s)
    all[n] = col
    if not s:match"X$" then
      t = s:find"[+-]$" and y or x
      t[1+#t] = col end end 
  return {all=all, x=x, y=y, names=row} end

 --     ._ _    _   _|_  |_    _    _|   _ 
 --     | | |  (/_   |_  | |  (_)  (_|  _> 

 -- add(i,v,inc) --> v ;; Update `i` with `v` (incrementing by `inc`).
 local function add(i,v,  inc)
  if v == "?" then return v end
  inc = inc or 1
  i.n = i.n + inc
  if i.has then i.has[v] = inc + (i.has[v] or 0) 
  elseif i.mu then 
    if inc < 0 and i.n < 2 then i.sd, i.m2, i.mu, i.n = 0,0,0,0 else
      local d = v - i.mu
      i.mu = i.mu + inc * d / i.n
      i.m2 = i.m2 + inc * d * (v - i.mu)
      i.sd = i.n<2 and 0 or sqrt((max(0,i.m2)/(i.n - 1))) end  
  elseif i.rows then  
    if not i.cols then i.cols = COLS(v) else
      i._mid = nil
      for _,col in pairs(i.cols.all) do add(col, v[col.at], inc) end 
      if inc > 0 then i.rows[1 + #i.rows] = v end end end
  return v end  

 -- sub(i,v) --> v ;; Decrement `v` from `i`.
 local function sub(i,v) return add(i,v,-1) end

 -- adds(src,it) --> it ;; Update `it` with all items from `src`.
 function adds(src, it)
  it = it or NUM()
  if type(src) == "string" 
  then for row in csv(src) do add(it,row) end
  else for _,row in pairs(src or {}) do add(it,row) end end
  return it end

 -- mid(i) --> v|row ;; Return central tendency of `i`.
 local function mid(i)--> a | v;; Exepcted value for `i`.
  if     i.mu   then return i.mu 
  elseif i.has  then return mode(i.has) 
  elseif i.rows then
    if not i._mid then
      local t={}; for _,col in pairs(i.cols.all) do t[1+#t] = mid(col) end 
      i._mid = t end
    return i._mid end end

 -- norm(i,v) --> n ;; Normalize `v` 0..1 using `i`.
 local function norm(i,v)
  return (i.has or v=="?") and v 
         or 1/(1 + math.exp(-1.7 * (v - i.mu)/(i.sd + 1e-32))) end

 -- aha(col,v1,v2) --> n ;; Return distance between `v1` and `v2`.
 local function aha(col,v1,v2)
  if v1=="?" and v2=="?" then return 1 end
  if col.has then return v1==v2 and 0 or 1 end
  v1,v2 = norm(col,v1), norm(col,v2)
  v1 = v1 ~= "?" and v1 or (v2 > 0.5 and 0 or 1)
  v2 = v2 ~= "?" and v2 or (v1 > 0.5 and 0 or 1)
  return abs(v1 - v2) end

 -- distx(i,row1,row2) --> n ;; Return distance  `row1` to `row2` (using X cols).
 local function distx(i,row1,row2,     d)
  d=0; for _,x in pairs(i.cols.x) do d= d + aha(x, row1[x.at],row2[x.at])^2 end
  return sqrt(d/#i.cols.x) end

 -- disty(i,row) --> n ;; Return distance of `row` to best goal (using Y cols).
 local function disty(i,row,     d)
  d=0; for _,y in pairs(i.cols.y) do d= d + (norm(y, row[y.at]) - y.best)^2 end
  return sqrt(d/#i.cols.y)  end

 -- distys(i,rows) --> rows ;; Sort `rows` by their distance to heaven.
 local function distys(i,  rows,      y)
   y = function(row) return disty(i, row) end
   return sort(rows or i.rows, function(r1,r2) return y(r1) < y(r2) end) end

 --     _|_  |_   o  ._   |  
 --      |_  | |  |  | |  |< 

 -- two(data) --> t ;; Incrementally cluster `data` into `best` and `rest`.
 local function two(data) 
  local train,test,start,todo,seen,best,rest,d
  shuffle(data.rows)
  train,test = cut(data.rows, data.n//2)
  start,todo = cut(train, 4)
  seen       = clone(data, start)
  best,rest  = cut(distys(seen),2,data)
  d          = function(row,what) return distx(seen, row, mid(what)) end
  for n,row in pairs(todo) do
    if n>256 then break end; --say(".")
    if d(row,best) < d(row,rest) then 
      add(seen, add(best, row))  ; --say(best.n)
      if best.n > sqrt(seen.n) then  -- print("-")
        add(rest, sub(best, table.remove( distys(best)))) end end end
  distys(best)
  return {best=best, rest=rest, seen=seen, test=test, 
          model = lt(function(row) return d(row,best) - d(row,rest) end)} end

 --      _|   _   ._ _    _    _ 
 --     (_|  (/_  | | |  (_)  _> 

 local egs={}

 egs["-h"] = function(_) print("\n"..help.."\n") end
 egs["-s"] = function(n) math.randomseed(n); the.seed =n end
 egs["--the"] = function(_) print(o(the)) end
 egs["--csv"] = function(_) for row in csv(the.file) do print(o(row)) end end
 egs["--shuffle"] = function(_) print(o(shuffle{10,20,30,40,50})) end
 egs["--mode"] = function(_)  print(mode{d=2,f=10,g=1})  end

 egs["--cut"] = function(_,    b,c) 
  b,c=cut({10,20,30,40,50},2); print(o(b),o(c)) 
  for _ =1,100 do b,c=cut({10,20,30,40,50},2) end end

 egs["--num"] = function(_,num)
  num=NUM()
  for _=1,1000 do add(num, box_muller(10,5)) end
  print(fmt("%.3f %.3f", num.mu, num.sd)) end 

 egs["--data"] = function(_) 
  for n,col in pairs(DATA(the.file).cols.x) do 
  print(n,o(col)) end end

 egs["--distx"]= function(_,    data,t,u)
  data = DATA(the.file)
  t,rows = {}, shuffle(data.rows)
  for n = 2,#rows do t[1+#t] = distx(data,rows[n-1],rows[n]) end
  print(o(sort(t))) end 

 egs["--disty"]= function(_,    data,num)
  data,t = DATA(the.file), {}
  distys(data)
  for n,row in pairs(data.rows) do t[n]=disty(data,row) end
  print(o(t)) end 

 egs["--inc"] = function(_, data1,data2)
  data1 = DATA(the.file)
  print(o(mid(data1)))
  data2 = clone(data1)
  for _,row in pairs(data1.rows) do
    add(data2,row)
    if data2.n==50 then print(o(mid(data2))) end end 
  while data2.rows do
    sub(data2, table.remove(data2.rows))
    if data2.n==50 then print(o(mid(data2)));break end end end

 egs["--two"] = function(_,    data,out,t)  
  t,data = {}, DATA(the.file)
  for _=1,20 do
    out = two(data)
    t[1+#t] = (100*disty(out.seen, sort(out.test, out.model)[1]))//1 end 
  print(o(sort(t))) end

 -- cli(d,funs) --> nil ;; Update `d` with flags from command-line; run `funs`.
 local function cli(d,funs)
  for i,s in pairs(arg) do
    if funs[s]
    then funs[s](coerce(arg[i+1]))
    else for k,_ in pairs(d) do
          if k:sub(1,1)==s:sub(2) then d[k]=coerce(arg[i+1]) end end end end end

 if arg[0]:find"two.lua" then cli(the,egs) end
diff --git a/two_data.5 b/two_data.5
 .\" Manpage for two.lua data format
 .\" Contact [email protected]
 .TH TWO_DATA 5 "November 29, 2025" "1.0" "File Formats"
 .SH NAME
 two_data \- input CSV format for the two.lua XAI tool
 .SH DESCRIPTION
 The \fBtwo.lua\fR script processes data stored in comma-separated value (CSV) files. The format relies heavily on a structured header row to define variable types, optimization goals, and ignored columns.
 .PP
 The file must be plain text, with rows separated by newlines and columns separated by commas.
 .SH HEADER SYNTAX
 The first row of the file is crucial. It defines the schema. The naming convention of the column headers tells the script how to treat the data in that column.
 .SS Data Types
 .TP
 .B Uppercase Start (e.g., \fIAge\fR, \fISalary\fR)
 Columns where the header name starts with an uppercase letter are treated as \fBNUM\fR (Numeric). The script will calculate mean and standard deviation for these.
 .TP
 .B Lowercase Start (e.g., \fIjob\fR, \fIrace\fR)
 Columns where the header name starts with a lowercase letter are treated as \fBSYM\fR (Symbolic). The script will calculate mode and entropy for these.
 .SS Column Roles
 .TP
 .B Suffix '+' (e.g., \fIClass+\fR, \fIAcc+\fR)
 Indicates a dependent variable (target) that should be \fBMAXIMIZED\fR. These columns constitute the "Y" (goal) variables.
 .TP
 .B Suffix '-' (e.g., \fILbs-\fR, \fICost-\fR)
 Indicates a dependent variable (target) that should be \fBMINIMIZED\fR. These columns constitute the "Y" (goal) variables.
 .TP
 .B Suffix 'X' (e.g., \fIidX\fR, \fIDateX\fR)
 Indicates a column that should be \fBIGNORED\fR entirely. These are often used for unique identifiers, comments, or raw data not suitable for clustering.
 .TP
 .B No Suffix
 Any column without a '+', '-', or 'X' suffix is treated as an independent variable ("X" variable) used for clustering and distance calculations.
 .SH DATA FORMAT
 .TP
 .B Numeric Values
 Standard integer or floating-point numbers.
 .TP
 .B Symbolic Values
 String identifiers. Note that the script splits strictly on commas, so strings containing commas may cause parsing errors.
 .TP
 .B Missing Values
 Missing data should be represented by the question mark character (\fB?\fR). The script contains logic to handle these during distance calculations (assumed max distance).
 .SH EXAMPLES
 .PP
 .B Example 1: A simple optimization dataset
 .PP
 In this example:
 .br
 - \fInameX\fR is ignored (ends in X).
 .br
 - \fIAge\fR is numeric independent (starts Upper).
 .br
 - \fIjob\fR is symbolic independent (starts lower).
 .br
 - \fISalary+\fR is a numeric goal to maximize.
 .PP
 .nf
 nameX,Age,job,Salary+
 1,25,engineer,50000
 2,30,doctor,90000
 3,?,artist,40000
 .fi
 .PP
 .B Example 2: Multi-objective car selection
 .PP
 In this example:
 .br
 - \fIClndrs\fR, \fIVol\fR, \fIHp\fR are numeric inputs.
 .br
 - \fIorigin\fR is a symbolic input.
 .br
 - \fILbs-\fR is a goal to minimize (weight).
 .br
 - \fIAcc+\fR is a goal to maximize (acceleration).
 .PP
 .nf
 Clndrs,Vol,Hp,origin,Lbs-,Acc+
 8,304,193,1,4732,18.5
 8,360,215,1,4615,14
 4,97,52,3,2130,24.6
 .fi
 .SH SEE ALSO
 .BR two.lua (1)
diff --git a/zenOfBash.md b/zenOfBash.md
diff --git a/zenOfGawk.md b/zenOfGawk.md
diff --git a/zenOfMake.md b/zenOfMake.md
	#!/usr/bin/env gawk -f
	# Copyright (c) 2025 Tim Menzies, MIT License
	# https://opensource.org/licenses/MIT
	BEGIN { FS=" " }
	{ seen($1,$2) }
	END { report() }

	function seen(want,got, kl) {
	if(!(want in cf)) { cf[want]["tn"] = total }
	if(!(got in cf)) { cf[got]["tn"] = total }
	total++
	for(kl in cf) {
	if(kl==want) { cf[kl]["tp"]+=(got==want); cf[kl]["fn"]+=(got!=want) }
	else { cf[kl]["fp"]+=(got==kl); cf[kl]["tn"]+=(got!=kl) }}}

	function report( kl,a,b,c,d) {
	print " N TN FN FP TP PD PREC PF ACC LABEL"
	for(kl in cf) {
	a = cf[kl]["tn"]; b = cf[kl]["fn"]
	c = cf[kl]["fp"]; d = cf[kl]["tp"]
	printf "%5d %5d %5d %5d %5d %5d %5d %5d %5d %-20s\n",
	d+b, a, b, c, d,
	div(d,d+b), div(d,c+d), div(c,c+a), div(d+a,a+b+c+d), kl \| "sort -n" }}

	function div(a,b) { return int(100*a/(b+1E-32)) }
	#!/usr/bin/env lua
	local help = [[
	bins.lua : stochastic incremental XAI
	(c) 2025, Tim Menzies, [email protected], mit-license.org

	Options:
	-h Show help.
	-b bins=7 Number of bins for discretization.
	-e era=30 Update model every `era` number of rows.
	-r ruleMax=3 Max conditions in a rule.
	-s seed=42 Random number seed.
	-f file=../lua6/auto93.csv ]]

	local function coerce(s)
	if s then return tonumber(s) or s:match'^%s(.-)%s$' end end

	local the={}; for k,v in help:gmatch("(%S+)=(%S+)") do the[k] = coerce(v) end
	math.randomseed(the.seed)

	local DATA, NUM, SYM, COLS, clone, adds
	local abs,exp,sqrt,log = math.abs, math.exp, math.sqrt, math.log
	local max,rand,cos = math.max, math.random, math.cos

	local fmt = string.format
	local sort = function(t,f) table.sort(t,f); return t end
	local lt = function(f) return function(a,b) return f(a) < f(b) end end
	local cat = function(a) return "{".. table.concat(a," ") .."}" end

	local function o(v, list,dict)
	list = function(a, u)
	for _,v in ipairs(a) do u[1+#u] = o(v) end; return cat(u) end
	dict = function(d, u)
	for k,v in pairs(d) do u[1+#u] = fmt(":%s %s", k, o(v)) end
	return cat(sort(u)) end
	return type(v) == "number" and fmt(v%1==0 and "%.0f" or "%.3f", v) or
	type(v) ~= "table" and tostring(v) or (#v>0 and list or dict)(v,{}) end

	local function s2a(s, a)
	a={}; for s1 in s:gmatch"([^,]+)" do a[1+#a] = coerce(s1) end; return a end

	local function csv(file, src)
	src = assert(io.open(file))
	return function( s)
	s = src:read()
	if s then return s2a(s) else src:close() end end end

	local function cut(a0,n, data)
	local a1,a2 = {},{}
	for j,v in ipairs(a0) do if j <= n then a1[1+#a1]=v else a2[1+#a2]=v end end
	if data then return clone(data,a1),clone(data,a2) end
	return a1,a2 end

	function box_muller(mu,sd)
	return mu + sd * sqrt(-2 * log(rand())) * cos(6.28 * rand()) end

	-- ----------------------------------------------------------------------------
	function DATA( src) return adds(src, {n=0,rows={},cols=nil}) end

	function clone(i, src) return adds(src, DATA{i.cols.names}) end

	function NUM(at,s)
	return {at=at or 0, of=s, n=0, mu=0, m2=0, sd=0, bins={},
	best=(tostring(s) or ""):find"+$" and 1 or 0} end

	function SYM(at,s) return {at=at, of=s, n=0, has={}, bins={}} end

	function COLS(row, t,x,y,all,col)
	x,y,all = {},{},{}
	for n,s in ipairs(row) do
	col = (s:match"^[A-Z]" and NUM or SYM)(n,s)
	all[n] = col
	if not s:match"X$" then
	t = s:find"[+-]$" and y or x
	t[1+#t] = col end end
	return {all=all, x=x, y=y, names=row} end

	local function add(i,v)
	if v == "?" then return v end
	i.n = i.n + 1
	if i.has then i.has[v] = 1 + (i.has[v] or 0)
	elseif i.mu then
	local d = v - i.mu
	i.mu = i.mu + d / i.n
	i.m2 = i.m2 + d * (v - i.mu)
	i.sd = i.n<2 and 0 or sqrt((i.m2/(i.n - 1)))
	elseif i.rows then
	if not i.cols then i.cols = COLS(v) else
	for _,col in pairs(i.cols.all) do add(col, v[col.at]) end
	i.rows[1 + #i.rows] = v end end
	return v end

	function adds(src, it)
	it = it or NUM()
	if type(src) == "string"
	then for row in csv(src) do add(it,row) end
	else for _,row in pairs(src or {}) do add(it,row) end end
	return it end

	local function norm(i,v)
	return (i.has or v=="?") and v
	or 1/(1 + math.exp(-1.7 * (v - i.mu)/(i.sd + 1e-32))) end

	local function disty(i,row, d)
	d=0; for _,y in pairs(i.cols.y) do d= d + (norm(y, row[y.at]) - y.best)^2 end
	return sqrt(d/#i.cols.y) end

	local function distys(i, rows, y)
	y = function(row) return disty(i, row) end
	return sort(rows or i.rows, function(r1,r2) return y(r1) < y(r2) end) end

	local function two(data)
	local train,test,start,todo,seen,best,rest,d
	shuffle(data.rows)
	train,test = cut(data.rows, data.n//2)
	start,todo = cut(train, 4)
	seen = clone(data, start)
	best,rest = cut(distys(seen),2,data)
	d = function(row,what) return distx(seen, row, mid(what)) end
	for n,row in pairs(todo) do
	if n>256 then break end
	if d(row,best) < d(row,rest) then
	add(seen, add(best, row))
	if best.n > sqrt(seen.n) then
	add(rest, sub(best, table.remove( distys(best)))) end end end
	distys(best)
	return {best=best, rest=rest, seen=seen, test=test,
	model=function(row) return d(row,best) - d(row, rest) end} end

	-- ----------------------------------------------------------------------------
	local egs={}

	egs["-h"] = function(_) print("\n"..help.."\n") end
	egs["-s"] = function(n) math.randomseed(n); the.seed =n end
	egs["--the"] = function(_) print(o(the)) end
	egs["--csv"] = function(_) for row in csv(the.file) do print(o(row)) end end

	egs["--num"] = function(_,num)
	num=NUM()
	for _=1,1000 do add(num, box_muller(10,5)) end
	print(fmt("%.3f %.3f", num.mu, num.sd)) end

	egs["--data"] = function(_)
	for n,col in pairs(DATA(the.file).cols.x) do
	print(n,o(col)) end end

	egs["--disty"]= function(_, data,num)
	data,t = DATA(the.file), {}
	distys(data)
	for n,row in pairs(data.rows) do t[n]=disty(data,row) end
	print(o(t)) end

	local function cli(d,funs)
	for i,s in pairs(arg) do
	if funs[s]
	then funs[s](coerce(arg[i+1]))
	else for k,_ in pairs(d) do
	if k:sub(1,1)==s:sub(2) then d[k]=coerce(arg[i+1]) end end end end end

	if arg[0]:find"bins.lua" then cli(the,egs) end
	#!/usr/bin/env bash
	# Copyright (c) 2025 Tim Menzies, MIT License
	# https://opensource.org/licenses/MIT
	alias ls="\ls --color"
	alias reload="source '$Here/ell' && echo ✅"
	alias grep='grep --color=auto'
	alias tree='tree -C'

	export BASH_SILENCE_DEPRECATION_WARNING=1
	export PATH="$Here:$PATH"
	export HISTSIZE=10000
	export HISTFILESIZE=20000
	export HISTCONTROL=ignoredups:erasedups

	Here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
	branch() { git branch 2>/dev/null \| awk '/^\*/ {print $2}'; }
	dirty() { [[ -n $(git status -s 2>/dev/null) ]] && echo "*"; }
	bold=$(tput bold) col0=$(tput sgr0) col1=$(tput setaf 6) col2=$(tput setaf 3)

	PROMPT_COMMAND='PS1="${bold}${col1}$(basename "$(dirname "$PWD")")/$(basename "$PWD")${col0} ${col2}$(branch)$(dirty)${col0} ▶ "'

	vi() {
	nvim --clean \
	--cmd "let g:netrw_banner=0 \| let g:netrw_liststyle=3 \| let g:netrw_browse_split=4 \| let g:netrw_winsize=15" \
	--cmd "set number relativenumber cursorline mouse=a clipboard=unnamedplus ignorecase smartcase" \
	--cmd "set statusline=%#StatusLine#\ ▶\ %f\ %m%r%=%y\ ❖\ %l:%c\ ❖\ %p%%\ " \
	--cmd "set expandtab tabstop=2 shiftwidth=2 splitright splitbelow" \
	--cmd "set undofile undodir=~/.vim/undo" \
	--cmd "nnoremap Q :quitall<CR>" \
	--cmd "colorscheme zaibatsu" \
	--cmd "set laststatus=2" \
	"$@"
	}

	hi() {
	clear
	echo "${col1}"
	cat<<'EOF'
	██╗ ██╗ ████████╗ ██████╗
	██║ ██║ ╚══██╔══╝ ██╔══██╗
	██║ ██║ ██║ ██████╔╝
	██║ ██║ ██║ ██╔══██╗
	███████╗ ██║ ██║ ██║ ██║
	╚══════╝ ╚═╝ ╚═╝ ╚═╝ ╚═╝
	(cause simple ain't stupid)
	EOF
	echo "${col0}"
	}

	inst() {
	local m=""
	for p in $1; do command -v "$p" &>/dev/null \|\| m+="$p "; done
	[ "$m" ] && case "$(uname -s)" in
	Darwin*) brew install $m ;;
	Linux*) sudo apt install -y $m ;;
	MINGW*) winget install $m ;;
	esac
	}

	# only run slow or verbose command at initial startup, not on reload
	if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
	inst git nvim gawk tree
	hi
	exec bash --init-file "${BASH_SOURCE[0]}" -i
	fi
	#!/usr/bin/env sh
	figlet -W -f mini $@ \| gawk '{print "-- " $0}'
	# Lua.ssh --- Sheet definitions for Lua source code
	# Copyright (c) 2014 Kenji Rikitake
	# Copyright (c) 1999 Edward Arthur, Akim Demaille, Miguel Santana
	#

	#
	# This file is NOT a part of a2ps.
	#
	# This program is free software; you can redistribute it and/or modify
	# it under the terms of the GNU General Public License as published by
	# the Free Software Foundation; either version 2, or (at your option)
	# any later version.
	#
	# This program is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	# GNU General Public License for more details.
	#
	# You should have received a copy of the GNU General Public License
	# along with this program; see the file COPYING. If not, write to
	# the Free Software Foundation, 59 Temple Place - Suite 330,
	# Boston, MA 02111-1307, USA.

	## This style is derived from Edward Arthur's AWK Style Sheet

	style Lua is

	written by "Kenji Rikitake <[email protected]>"
	version is 0.1
	requires a2ps version 4.9.7

	documentation is
	"This style file is intended to support the Lua programming language source code."
	end documentation

	alphabets are
	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_0"
	case sensitive

	keywords in Keyword are
	and, false, ipairs, nil, not, or, pairs, rawequal, rawget,
	rawlen, rawset, select, tonumber, tostring, true, type
	end keywords

	#keywords in Keyword_strong are
	keywords in Keyword_strong are
	assert, break, collectgarbage, do, dofile, else, elseif, "end",
	error, for, function, getmetatable, goto, if, "in", local,
	load, loadfile, next, pcall, print, repeat, require, return,
	setmetatable, then, until, while, xpcall, _G, _VERSION
	end keywords

	keywords in Label_strong are
	"^" function
	end keywords

	keywords in Comment are
	self
	end keywords

	sequences are
	"[[" Comment "]]",
	"--" Comment,
	C-string
	end sequences
	end style
	# vim: ts=2 sw=2 noet
	# Copyright (c) 2025 Tim Menzies, MIT License
	# https://opensource.org/licenses/MIT
	#------------------------------------------------------
	K ?= 1#
	M ?= 2#
	BINS ?= 4#
	ERA ?= 100#
	DATA = ~/gits/moot/classify/diabetes.csv#
	DATAG = ~/gits/moot/optimize/misc/auto93.csv#

	G = gawk -f#
	OK = gawk -f ok.awk -f#

	#------------------------------------------------------
	SHELL=/bin/bash
	export PATH := $(CURDIR):$(PATH)
	.SILENT:
	.PHONY: help egs ok eg-nbc eg-abcd eg-soybean eg-globals pull push

	help: ## show this help
	@gawk 'BEGIN { FS=":.*?## ";c="\033[1;3"; r="\033[0m"; \
	printf "\n%s6mmake%s [%s3moptions%s]:\n\n",c,r,c,r} \
	NF==2 && $$1~/^[a-z0-9A-Z_-]+/{ \
	printf " %s2m%-15s%s %s\n",c,$$1,r,$$2}' $(MAKEFILE_LIST)

	#------------------------------------------------------
	hi=@echo -e "\n--------- $@"

	test: eg-nbc eg-abcd eg-soybean eg-globals ## run all egs: now

	ok:; chmod +x nbc abcd bins

	eg-bins: ok $(DATA) ## run naive bayes classifier
	cat $(DATA) \| bins BINS=$(BINS)

	eg-nbc: ok $(DATA) ## run naive bayes classifier
	$(hi); cat $(DATA) \| bins BINS=$(BINS) \| nbc K=$K M=$M \| sort \| uniq -c

	eg-abcd: ok $(DATA) ## run classifier with confusion matrix
	$(hi); cat $(DATA) \| bins BINS=$(BINS) \
	\| nbc K=$K M=$M #\| abcd

	eg-soybean: ## run classifier on soybean
	$(hi); $(MAKE) eg-abcd DATA=~/gits/timm/moot/classify/soybean.csv

	eg-best: ok $(DATA) ## run naive bayes classifier
	cat $(DATAG) \| bins BINS=$(BINS) \| best \| column -s, -t \| sed -n '1,5p;395,$$p'

	eg-tree: ok $(DATA) ## run naive bayes classifier
	cat $(DATAG) \| bins BINS=$(BINS) \| best \| tree

	eg-globals: ok $(DATA) ## run classifier with confusion matrix
	$(hi); ( cat $(DATA) \| $(OK) bins BINS=$(BINS) \
	\| $(OK) nbc K=$K M=$M \| $(OK) abcd ) > /dev/null

	# misc ------------------------------------------------
	sh: ## run a customized shell
	sh ell

	pull: ## update from main
	git pull

	push: ## commit to main
	git commit -am saving; git push; git status

	~/tmp/%.pdf: %.lua
	echo "pdf-ing $@ ... "
	a2ps \
	--file-align=virtual \
	--line-numbers=1 \
	--pro=color \
	--lines-per-page=120 \
	--pretty=lua.ssh \
	--left-title="" \
	--borders=no \
	--right-footer="page %s. of %s#" \
	--landscape \
	--columns 3 \
	-M letter \
	-o - $^ \| ps2pdf - $@
	open $@
	# Copyright (c) 2025 Tim Menzies, MIT License
	# https://opensource.org/licenses/MIT
	END { rogues() }

	function rogues( i,s,known) {
	known = "^(NF\|NR\|FS\|RS\|RT\|FNR\|OFS\|ORS\|" \
	"PREC\|ARGC\|ARGV\|OFMT\|LINT\|FPAT\|" \
	"ERRNO\|RSTART\|ARGIND\|" \
	"SUBSEP\|CONVFMT\|ENVIRON\|SYMTAB\|FUNCTAB\|PROCINFO\|" \
	"FILENAME\|RLENGTH\|BINMODE\|" \
	"IGNORECASE\|FIELDWIDTHS\|ROUNDMODE\|TEXTDOMAIN)$"
	for(i in SYMTAB) if (i !~ known) s = s " " i
	if (s) print "?" s > "/dev/stderr" }
Feature	Raw `print()`	Custom `say()` Wrapper	Standard `logging`
Control	None	High (Custom code)	High (Configuration)
Silencing	Manual deletion	Single Flag (All or Nothing)	Leveled (Debug vs Error)
Complexity	Low	Low	Medium
Best For	Throwaway code	Scripts & Tools	Production Apps
	#!/usr/bin/env lua
	local the,l,egs = {},{},{} -- config, lib, demos
	local help = [[
	six : stochastic incremental XAI
	(c) 2025, Tim Menzies, [email protected], mit-license.org

	Options:
	-h Show help.
	-b bins=7 Number of bins for discretization.
	-e era=30 Update model every `era` number of rows.
	-r ruleMax=3 Max conditions in a rule.
	-s seed=42 Random number seed.
	-f file=../lua6/auto93.csv ]]

	-- ----------------------------------------------------------------------------
	local DATA, NUM, SYM, COLS = {},{},{},{}

	function DATA.new( rows, i)
	i = l.new(DATA,{rows={}, cols=nil})
	for _,row in pairs(rows or {}) do i:add(row) end
	return i end

	function DATA.clone(i, rows, clone)
	clone = DATA:new({i.cols.names})
	for _,row in pairs(rows or {}) do clone:add(row) end
	return clone end

	function NUM.new(at,s)
	return l.new(NUM, {at=at or 0, of=s, n=0, mu=0, m2=0, sd=0, bins={},
	best=(tostring(s) or ""):find"+$" and 1 or 0}) end

	function SYM.new(at,s)
	return l.new(SYM, {at=at, of=s, n=0, bins={}, has={}, mode=0, most=-1}) end

	function COLS.new(row, x,y,all,col)
	x,y,all = {},{},{}
	for n,s in ipairs(row) do
	col = (s:match"^[A-Z]" and Num or Sym).new(n,s)
	all[n] = col
	if not s:match"X$" then
	l.push(s:find"[+-]$" and y or x, col) end end
	return l.new(COLS, {all=all, x=x, y=y, names=row}) end

	-- -----------------------------------------------------------------------------
	function DATA.add(i,row)
	if not i.cols then i.cols = COLS.new(row) else
	for _,col in pairs(i.cols.all) do col:add(row[col.at]) end
	l.push(i.rows, row) end
	return row end

	function SYM.add(i,v)
	if v~="?" then
	i.n = i.n + 1
	i.has[v] = 1 + (i.has[v] or 0)
	if i.has[v] > i.most then i.most,i.mode = i.has[v],v end end
	return v end

	function NUM.add(i,n, d)
	if v~="?" then
	i.n = i.n + 1
	d = n - i.mu
	i.mu = i.mu + d / i.n
	i.m2 = i.m2 + d * (n - i.mu)
	i.sd = i.n<2 and 0 or (i.m2/(i.n - 1))^0.5 end
	return n end

	-- -----------------------------------------------------------------------------
	function NUM.norm(i,v)
	return 1/(1 + math.exp(-1.7 * (v - i.mu)/(i.sd + 1e-32))) end

	function SYM.bin(_,v) return v end
	function NUM.bin(i,v) return v=="?" and v or math.floor(the.bins*i:norm(v)) end

	function DATA.disty(i,row, d)
	d=0; for _,y in pairs(i.cols.y) do d= d + (y:norm(row[y.at]) - y.best)^2 end
	return (d/#(i.cols.y)) ^ 0.5 end

	function DATA.bins(i,row, v,y)
	y = i:disty(row)
	for _,x in pairs(i.cols.x) do
	v = x:bin(row[x.at])
	if v ~= "?" then
	x.bins[v] = x.bins[v] or Num.new(x.at,v)
	x.bins[v]:add(y) end end end

	-- -----------------------------------------------------------------------------
	function DATA.rule(i,stop, t,u,f,stop)
	f = function(n) return math.floor(100*(n.mu + n.sd/n.n^.5)) end
	t,u = {},{}
	for _,x in pairs(i.cols.x) do
	for _,col in pairs(x.bins) do
	push(t, col) end end
	for j,col in pairs(sort(t, lt(f))) do
	if stop and j > stop then break end
	u[col.at] = u[col.at] or {}
	push(u[col.at], col) end -- tests on each attribute are disjunctions
	return u end

	function DATA.or(i,ors,row)
	for _, col in pairs(ors) do
	if col.of == i.cols.all[col.at]:bin(row[col.at]) then return true end end
	return false end

	function DATA.and(i,ands,row)
	for _,ors in pairs(ands) do if not i:or(ors,row) then return false end end
	return true end

	function DATA.selects(i,rule,rows, u)
	u={}; for _,r in pairs(rows) do if i:and(i,rule,r) then l.push(u,r) end end
	return u end

	function main(file, data1,y,train,test,rule)
	data1 = Data.new()
	for _,row in csv(file) do data1:add(row) end
	train, test = {},{}
	for j,row in pairs(shuffle(data1.rows)) do
	push(j < #data.rows/2 and train or test, row) end
	rule,data2 = nil, data1:clone()
	y = function(row) return data2:disty(row) end
	for r,row in pairs(train) do
	data2:bins( data2:add(row))
	if r % the.era == 0 then
	rule = data2.rules( math.min(3, #data2.rows/the.era) )
	print(Y(sort(data2:selects(rule,test),Y)[1])) end end end

	-- -----------------------------------------------------------------------------
	-- Misc tricks
	function l.new(meta,t) meta.__index=meta; return setmetatable(t,meta) end

	l.fmt = string.format

	function l.push(t,v) t[1+#t]=v; return v end
	function l.sort(t,f) table.sort(t,f); return t end
	function l.lt(f) return function(a,b) return f(a) < f(b) end end

	function l.shuffle(t, n)
	for m= #t,2,-1 do n=math.random(m); t[m],t[n] = t[n],t[m] end
	return t end

	-- Thing to string
	local function _a2a(a,u) for _,v in ipairs(a) do push(u,cat(v)) end;return u end

	local function _d2a(d,u)
	for k,v in pairs(d) do
	if type(v)~=type(_d2a) then
	l.push(u,l.fmt(":%s %s",k,type(v)==type(_d2a) and "" or l.cat(v))) end end
	return l.sort(u) end

	function l.cat(t)
	return type(t) == "number" and l.fmt(t%1==0 and "%s" or "%.3f", t)
	or type(t) ~= "table" and tostring(t)
	or "{".. table.concat((#t>0 and _a2a or _d2a)(t,{})," ") .."}" end

	function l.pat(t) print(l.cat(t)); return t end

	-- String to thing
	function l.coerce(s)
	if s==nil then return nil end
	return tonumber(s) or s:match'^%s(.-)%s$' end

	local function _cells(s, a)
	a={}; for s1 in s:gmatch"([^,]+)" do l.push(a, l.coerce(s1)) end; return a end

	function l.csv(file, n,stream)
	n,stream = 0,assert(io.open(file))
	return function( s)
	s= stream:read()
	if s then n=n+1; return n,_cells(s) else stream:close() end end end

	-- Config management.
	function l.settings(s, d)
	d={}; for k,v in s:gmatch("(%S+)=(%S+)") do d[k]=l.coerce(v) end; return d end

	function l.cli(d,funs)
	for i,s in pairs(arg) do
	if funs[s]
	then funs[s](l.coerce(arg[i+1]))
	else for k,_ in pairs(d) do
	if k:sub(1,1)==s:sub(2) then d[k]=l.coerce(arg[i+1]) end end end end
	return d end

	-- -----------------------------------------------------------------------------
	egs["-h"] = function(_) print("\n"..help.."\n") end
	egs["-s"] = function(n) math.randomseed(n); the.seed =n end
	egs["--the"] = function(_) l.pat(the) end
	egs["--guess"] = function(_) main(the.file) end

	the = l.settings(help)
	math.randomseed(the.seed)
	if arg[0]:find"six.lua" then the=l.cli(the,egs) end

	return {DATA=DATA, NUM=NUM, SYM=SYM, COLS=COLS, help=help, the=the, l=l}
	#!/usr/bin/env lua
	local the = {leaf=2}
	local push, coerce, cells, csv, inc2, cli
	local tree, show, read,_tally, _split, _div, _kids

	function tree(goal,klass,rows, col,val,op)
	local col1, val1, mu = _split(goal,klass, rows)
	return {rows=rows, col=col, val=val, op=op, mu= mu,
	kids=_kids(goal,klass,rows,col1,val1)} end

	function _split(goal,klass, rows, all,good,best,col,val,n,p,s,mu)
	all, good, mu = _tally(goal,klass, rows,{},{})
	best = -1
	for c, vals in pairs(good) do
	for v, g in pairs(vals) do
	n = all[c][v]
	p = g/n
	s = p*p / (p + (n-g)/n + 1e-32)
	if s > best then best, col, val = s, c, v end end end
	return col, val, mu end

	function _tally(goal,klass,rows,all,good, y)
	y = 0
	for _, row in pairs(rows) do
	y = y + row[klass]
	for c, v in pairs(row) do
	if c ~= klass and v ~= "?" then
	inc2(all, c, v)
	if row[klass] <= goal then inc2(good, c, v) end end end end
	return all, good, y/#rows end

	function _kids(goal,klass,rows,col1,val1, out)
	out = {}
	for op, t in pairs(_div(col1, val1, rows)) do
	if #t >= the.leaf and #t < #rows then
	out[op] = tree(goal,klass, t, col1, val1, op) end end
	return out end

	function _div(c,v,rows, yes,no)
	yes, no = {}, {}
	for _, r in pairs(rows) do push((r[c]=="?" or r[c]==v) and yes or no, r) end
	return {["=="] = yes, ["!="] = no} end

	----------------------------------------------------------------
	function show(t, lvl)
	print(string.format("%s%s (n=%d)", ("\|.. "):rep(lvl or 0),
	t.col and (t.col.." "..t.op.." "..t.val) or "", #t.rows))
	for _, kid in pairs(t.kids or {}) do
	show(kid, (lvl or 0) + 1) end end

	function read( rows,klass)
	rows = {}
	for row in csv() do
	if not klass then
	print(row)
	for i, s in pairs(row) do if s:find"-$" then klass = i end end
	else rows[#rows + 1] = row end end
	table.sort(rows, function(t,u) return t[klass] < u[klass] end)
	return rows[math.floor((#rows)^.5)][klass], klass, rows end

	-------------------------------------------------------------------------------
	function push(t,x) t[#t + 1] = x; return x end
	function coerce(s) return tonumber(s) or s end
	function cells(s, t)
	t = {}; for s1 in s:gmatch"([^,]+)" do t[#t + 1] = coerce(s1) end
	return t end

	function csv()
	return function() local s = io.read(); return s and cells(s) end end

	function inc2(t,c,v)
	t[c]=t[c] or {}; t[c][v]=(t[c][v] or 0)+1; return t[c][v] end

	function cli()
	for i, s in pairs(arg or {}) do
	if the[s:sub(2)] then the[s:sub(2)] = coerce(arg[i+1]) end end end

	-------------------------------------------------------------------------------
	if ... == nil then
	cli()
	local goal,klass,rows = read()
	show(tree(goal, klass, rows)) end

	return {tree=tree, show=show, read=read}



	"""

	def mad(rows):
	a = [y(row) for row in rows]
	median = sorted(a)[len(a) // 2]
	return sorted([abs(x - median) for x in a])[len(a) // 2]

	def mads(a,b): n1,n2=len(a),len(b); return (n1mad(a) + n2mad(b))/(n1+n2)

	def key(rows,col,klass):
	x = lambda row:row[col]
	y = lambda row:row[klass]
	rows = sorted([row in rows if x(row) != "?"]), key=x)
	most = mad(rows)
	b4,min = [],[],-1,len(rows)**.5
	for i in range(len(rows)):
	b = int(BINS/(1+e^(-1.704*(mu - x(rows[i])/sd))
	left += [row]
	if b != b4 and i>min and len(rows) - i > min:
	mad1,mad2 = mads(left),mads(rows[i:])
	if now < most
	=
	b4 = b

	for row in sorted(rows, key=lambda row

	function span(t,i, j, u)
	u,j = {},j or #t; for k=i,j do u[1+#u] = t[k] end; return u end

	function cut(rows,col,klass)
	x = function(r) return r[col] end
	y = function(r) return r[klass] end
	lt = function(r1,r2) return x(r1) < x(r2) end
	t = {}; for _,r in pairs(rows) if x(r) != "?" then t[1+#t]=r end end
	table.sort(t,lt)
	most = sd(t,y)
	for i=1,#t do
	j=#t-i
	b = (the.BINS/(1+math.exp(-1.7404*math.exp((mu- x(t[i]))/sd))))//2
	left[1 + #left] = rows[i]
	if b ~= b4 and i>min and j > min:
	sd1,sd2 = sd(span(rows,1,i),y), sd(span(rows,i+1))) / (i+j)
	now = (i * sd1 + j * sd2) / (i+j)
	if now < least then
	out={now, {"<=",col,x(rows[i])}, {">",col,x(rows[i+1])}



	function sd(row,fn, v,d,n,mu,m2)
	n,mu,m2 = 0,0,0
	for i=1,#t do
	v = fn(t[i])
	if v~="?" then
	n,d = n+1, v-mu
	mu = mu+d/n; m2 = m2+d*(x-mu) end end
	return n<2 and 0 or (m2/(n-1))^.5 or 0 end