Build your own tiny GPT

final class TinyGPT
{
    public array $vocab = [];   // token ID → the text of that token
    public array $ids = [];     // the text of a token → its ID
    public int $V;              // how many different tokens it knows
    public array $w = [];       // every weight, by name. Each one is a table of rows.
    private array $m = [];      // Adam: running average of each weight's gradient
    private array $s = [];      // Adam: running average of each gradient squared
    public int $steps = 0;      // training steps done so far

    public function __construct(string $text, public string $unit = 'letter', public int $T = 32, public int $d = 16, int $seed = 1)
    {
        // The vocabulary is every different token in the training text, sorted.
        $pieces = array_values(array_unique($this->split($text)));
        sort($pieces, SORT_STRING);
        $this->vocab = $pieces;
        $this->ids = array_flip($pieces);
        $this->V = count($pieces);

        seed_rng($seed);
        $d = $this->d; $h = 4 * $d;
        $shapes = [                         // [rows, columns, size of the random starting values]
            'tok'  => [$this->V, $d, 0.3],  // token embedding: d numbers for every token
            'pos'  => [$this->T, $d, 0.3],  // position embedding: d numbers for every place in the window
            'q'    => [$d, $d, 1 / sqrt($d)], // attention: query, key, value and output matrices
            'k'    => [$d, $d, 1 / sqrt($d)],
            'v'    => [$d, $d, 1 / sqrt($d)],
            'o'    => [$d, $d, 1 / sqrt($d)],
            'up'   => [$d, $h, 1 / sqrt($d)], // feed-forward: widen to 4×d neurons, ReLU, shrink back to d
            'upb'  => [1, $h, 0],
            'dn'   => [$h, $d, 1 / sqrt($h)],
            'dnb'  => [1, $d, 0],
            'out'  => [$d, $this->V, 1 / sqrt($d)], // a score for every token in the vocabulary
            'outb' => [1, $this->V, 0],
        ];
        foreach ($shapes as $name => [$rows, $cols, $scale]) {
            for ($i = 0; $i < $rows; $i++) for ($j = 0; $j < $cols; $j++) $this->w[$name][$i][$j] = gauss() * $scale;
            $this->m[$name] = $this->s[$name] = array_fill(0, $rows, array_fill(0, $cols, 0.0));
        }
    }

    // TOKENS (lesson 9): either single letters, or whole words and punctuation marks.
    public function split(string $text): array
    {
        if ($this->unit === 'letter') return preg_split('//u', $text, -1, PREG_SPLIT_NO_EMPTY) ?: [];
        preg_match_all("/\n|[\\p{L}\\p{N}']+|[^\\s\\p{L}\\p{N}']/u", $text, $m);
        return $m[0];
    }

    public function encode(string $text): array
    {
        $out = [];
        foreach ($this->split($text) as $piece) if (isset($this->ids[$piece])) $out[] = $this->ids[$piece];
        return $out;
    }

    public function decode(array $ids, string $before = ''): string
    {
        $s = $before;
        foreach ($ids as $i) {
            $piece = $this->vocab[$i];
            // words need their spaces put back, except before punctuation or after a new line
            $gap = $this->unit === 'word' && $s !== '' && !str_ends_with($s, "\n") && $piece !== "\n" && !preg_match('/^[.,!?;:]$/', $piece);
            $s .= ($gap ? ' ' : '') . $piece;
        }
        return substr($s, strlen($before));
    }

    public function paramCount(): int
    {
        $n = 0;
        foreach ($this->w as $W) $n += count($W) * count($W[0]);
        return $n;
    }

    /* ---------- small matrix helpers ---------- */
    private static function mul(array $x, array $W): array      // row of numbers × matrix
    {
        $out = array_fill(0, count($W[0]), 0.0);
        foreach ($x as $i => $xi) {
            if ($xi == 0.0) continue;
            foreach ($W[$i] as $j => $wij) $out[$j] += $xi * $wij;
        }
        return $out;
    }
    private static function mulT(array $g, array $W): array     // send a gradient back through W
    {
        $out = [];
        foreach ($W as $i => $row) {
            $sum = 0.0;
            foreach ($row as $j => $wij) $sum += $g[$j] * $wij;
            $out[$i] = $sum;
        }
        return $out;
    }
    private static function addOuter(array &$G, array $x, array $g): void   // G += every x[i] × g[j]
    {
        foreach ($x as $i => $xi) {
            if ($xi == 0.0) continue;
            $row = &$G[$i];
            foreach ($g as $j => $gj) $row[$j] += $xi * $gj;
            unset($row);
        }
    }
    private static function add(array $a, array $b): array { foreach ($b as $i => $v) $a[$i] += $v; return $a; }
    private static function softmax(array $z): array                   // scores → probabilities that add up to 1
    {
        $max = max($z); $sum = 0.0;
        foreach ($z as $i => $v) { $z[$i] = exp($v - $max); $sum += $z[$i]; }
        foreach ($z as $i => $v) $z[$i] = $v / $sum;
        return $z;
    }

    /* ---------- FORWARD PASS: tokens in, next-token probabilities out ---------- */
    public function forward(array $x, float $temperature = 1.0): array
    {
        $W = $this->w; $n = count($x); $scale = 1 / sqrt($this->d);
        $f = ['x' => $x];
        for ($t = 0; $t < $n; $t++) {
            // 1. EMBED: look up the token's row of numbers and add its position's row
            $e = self::add($W['tok'][$x[$t]], $W['pos'][$t]);
            $f['e'][$t] = $e;
            // 2. Every position makes a query ("what am I looking for?"),
            //    a key ("what do I contain?") and a value ("what I pass on").
            $f['q'][$t] = self::mul($e, $W['q']);
            $f['k'][$t] = self::mul($e, $W['k']);
            $f['v'][$t] = self::mul($e, $W['v']);
        }
        for ($t = 0; $t < $n; $t++) {
            // 3. ATTENTION: compare my query with the key of every EARLIER position
            //    (never later ones: that would be peeking at the answer), turn the
            //    scores into weights that add up to 1, and blend those positions' values.
            $score = [];
            for ($s = 0; $s <= $t; $s++) {
                $dot = 0.0;
                foreach ($f['q'][$t] as $i => $qi) $dot += $qi * $f['k'][$s][$i];
                $score[$s] = $dot * $scale;
            }
            $a = self::softmax($score);
            $mix = array_fill(0, $this->d, 0.0);
            foreach ($a as $s => $as) foreach ($f['v'][$s] as $i => $vi) $mix[$i] += $as * $vi;
            $f['a'][$t] = $a; $f['mix'][$t] = $mix;
            // 4. RESIDUAL: add what attention found on top of the original embedding
            $h1 = self::add($f['e'][$t], self::mul($mix, $W['o']));
            // 5. FEED-FORWARD: a small 2-layer network (lesson 6), run at every position
            $pre = self::add(self::mul($h1, $W['up']), $W['upb'][0]);
            $hid = array_map(fn($z) => $z > 0 ? $z : 0.0, $pre);          // ReLU
            $h2 = self::add($h1, self::add(self::mul($hid, $W['dn']), $W['dnb'][0]));
            // 6. OUTPUT: one score per token in the vocabulary → softmax → probabilities
            $logits = self::add(self::mul($h2, $W['out']), $W['outb'][0]);
            if ($temperature != 1.0) foreach ($logits as $i => $z) $logits[$i] = $z / $temperature;
            $f['h1'][$t] = $h1; $f['hid'][$t] = $hid; $f['h2'][$t] = $h2;
            $f['p'][$t] = self::softmax($logits);
        }
        return $f;
    }

    /* ---------- BACKWARD PASS: every weight's share of the blame (lesson 7) ---------- */
    private function backward(array $f, array $y, array &$G): float
    {
        $W = $this->w; $n = count($y); $scale = 1 / sqrt($this->d); $loss = 0.0;
        $dq = $dk = $dv = $de = [];
        for ($t = $n - 1; $t >= 0; $t--) {
            $loss -= log(max($f['p'][$t][$y[$t]], 1e-12));     // log loss: small when the right token got a high probability
            $dz = $f['p'][$t]; $dz[$y[$t]] -= 1.0;              // blame on each score = probability − (1 for the right token)
            self::addOuter($G['out'], $f['h2'][$t], $dz);
            $G['outb'][0] = self::add($G['outb'][0], $dz);
            $dh2 = self::mulT($dz, $W['out']);
            // back through the feed-forward network (dh2 also flows straight to h1: the residual)
            self::addOuter($G['dn'], $f['hid'][$t], $dh2);
            $G['dnb'][0] = self::add($G['dnb'][0], $dh2);
            $dhid = self::mulT($dh2, $W['dn']);
            foreach ($dhid as $j => $g) if ($f['hid'][$t][$j] <= 0) $dhid[$j] = 0.0;   // ReLU passes blame only where it was on
            self::addOuter($G['up'], $f['h1'][$t], $dhid);
            $G['upb'][0] = self::add($G['upb'][0], $dhid);
            $dh1 = self::add($dh2, self::mulT($dhid, $W['up']));
            // back through attention's output matrix
            self::addOuter($G['o'], $f['mix'][$t], $dh1);
            $dmix = self::mulT($dh1, $W['o']);
            $de[$t] = $dh1;
            // back through the blend: to each value, and to each attention weight
            $da = []; $dot = 0.0;
            foreach ($f['a'][$t] as $s => $as) {
                $g = 0.0;
                foreach ($dmix as $i => $dm) { $g += $dm * $f['v'][$s][$i]; $dv[$s][$i] = ($dv[$s][$i] ?? 0.0) + $as * $dm; }
                $da[$s] = $g; $dot += $as * $g;
            }
            // back through the softmax, then through the query·key scores
            foreach ($f['a'][$t] as $s => $as) {
                $ds = $as * ($da[$s] - $dot) * $scale;
                foreach ($f['k'][$s] as $i => $ki) {
                    $dq[$t][$i] = ($dq[$t][$i] ?? 0.0) + $ds * $ki;
                    $dk[$s][$i] = ($dk[$s][$i] ?? 0.0) + $ds * $f['q'][$t][$i];
                }
            }
        }
        // finally, back into the embeddings of each token and each position
        for ($t = 0; $t < $n; $t++) {
            foreach (['q' => $dq, 'k' => $dk, 'v' => $dv] as $name => $D) {
                if (!isset($D[$t])) continue;
                self::addOuter($G[$name], $f['e'][$t], $D[$t]);
                $de[$t] = self::add($de[$t], self::mulT($D[$t], $W[$name]));
            }
            $G['tok'][$f['x'][$t]] = self::add($G['tok'][$f['x'][$t]], $de[$t]);
            $G['pos'][$t] = self::add($G['pos'][$t], $de[$t]);
        }
        return $loss;
    }

    /* ---------- ONE TRAINING STEP ---------- */
    public function trainStep(array $data, int $batch, float $lr, int $seed): float
    {
        seed_rng($seed * 1000003 + $this->steps);   // step N always picks the same windows, however training is split up
        $G = [];
        foreach ($this->w as $name => $M) $G[$name] = array_fill(0, count($M), array_fill(0, count($M[0]), 0.0));
        $loss = 0.0;
        $ring = array_merge($data, array_slice($data, 0, $this->T + 1));   // wrap around, so the first lines get as much practice as the rest
        for ($b = 0; $b < $batch; $b++) {
            $start = mt_rand(0, count($data) - 1);
            $x = array_slice($ring, $start, $this->T);          // a window of the training text…
            $y = array_slice($ring, $start + 1, $this->T);      // …and the same window moved on by one: the right answers
            $loss += $this->backward($this->forward($x), $y, $G);
        }
        $n = $batch * $this->T;
        // ADAM: gradient descent with momentum (lesson 6), where each weight also
        // gets its own step size based on how big its gradients have been.
        $this->steps++;
        $b1 = 0.9; $b2 = 0.99;
        $c1 = 1 - $b1 ** $this->steps; $c2 = 1 - $b2 ** $this->steps;
        foreach ($G as $name => $rows) {
            foreach ($rows as $i => $row) {
                $wr = &$this->w[$name][$i]; $mr = &$this->m[$name][$i]; $sr = &$this->s[$name][$i];
                foreach ($row as $j => $g) {
                    $g /= $n;
                    $mr[$j] = $b1 * $mr[$j] + (1 - $b1) * $g;
                    $sr[$j] = $b2 * $sr[$j] + (1 - $b2) * $g * $g;
                    $wr[$j] -= $lr * ($mr[$j] / $c1) / (sqrt($sr[$j] / $c2) + 1e-8);
                }
                unset($wr, $mr, $sr);
            }
        }
        return $loss / $n;
    }

    /* ---------- WRITING: predict, pick one, add it to the text, repeat ---------- */
    public function generate(array $context, int $count, float $temperature, ?string $stopAt = null): array
    {
        $picked = [];
        if (!$context) $context = [0];                          // nothing it can read: start from the first token
        for ($c = 0; $c < $count; $c++) {
            $window = array_slice($context, -$this->T);        // it can only see the last T tokens
            $f = $this->forward($window, $temperature);
            $p = end($f['p']);                                  // probabilities for the token after the last one
            $r = rnd(); $next = count($p) - 1;                  // pick one at random, weighted by probability
            foreach ($p as $i => $pi) { $r -= $pi; if ($r <= 0) { $next = $i; break; } }
            $context[] = $next;
            $picked[] = [$next, $p[$next]];
            if ($stopAt !== null && $this->vocab[$next] === $stopAt) break;
        }
        return $picked;
    }

    public function export(): array { return ['w' => $this->w, 'm' => $this->m, 's' => $this->s, 'steps' => $this->steps]; }
    public function import(array $a): void { $this->w = $a['w']; $this->m = $a['m']; $this->s = $a['s']; $this->steps = (int) $a['steps']; }
}