OkayStudy
ZH English 한국어 日本語 中文 हिन्दी বাংলা اردو Español Português Français Deutsch Русский العربية Bahasa Indonesia Tiếng Việt ไทย Türkçe

如何开启通知

此浏览器或网址无法使用通知。请使用支持通知的浏览器，通过安全连接访问SONUB。

如果通知被屏蔽，请在浏览器设置中允许通知。

  1. 打开地址栏旁的站点设置或浏览器的站点设置，将此网站的通知设为允许。
  2. 在设备设置中允许浏览器发送通知，并检查勿扰模式或专注模式。
  3. 返回SONUB，刷新页面，然后通过铃铛开启通知。

在iPhone或iPad上，将SONUB添加到主屏幕，从主屏幕打开后允许通知。
· 从第一行到对话

现在，让我们打造属于你自己的人工智能吧。

一排词语积木的最后是一个空格。空格上方有三块候选积木，各带一根高低不同的柱子；其中一块被选中，接到这一排的末尾，一个箭头又绕回去，再选下一个。
5

GPT架构与代码

加；与计算关联，按混合Value。

只有一头一块，keys只含当前及过去位置，不能偷看未来。、、两层和输出头共同工作。为便于阅读，使用、无参数归一化，不使用。

动手输入并实验

先写engine.py并检查，再写model.py。Value记录局部导数，backward逆序累加影响。

cd ~/xslm
nano engine.py
nano model.py
python3 engine.py
python3 -c "from model import GPT; m=GPT(10, 8, 12); print(len(m.forward([1, 2])), len(m.forward([1, 2])[0]))"
在终端中练习
engine.py需要亲手编写的完整源码

为数值，grad为，parents保存局部导数。把分数变概率。

import math


class Value:
    def __init__(self, data, parents=()):
        self.data = float(data)
        self.grad = 0.0
        self.parents = parents

    def __add__(self, other):
        other = other if isinstance(other, Value) else Value(other)
        return Value(self.data + other.data, ((self, 1), (other, 1)))

    def __mul__(self, other):
        other = other if isinstance(other, Value) else Value(other)
        return Value(self.data * other.data,
                     ((self, other.data), (other, self.data)))

    def __pow__(self, power):
        return Value(self.data ** power,
                     ((self, power * self.data ** (power - 1)),))

    def __neg__(self):
        return self * -1

    def __sub__(self, other):
        return self + (-other)

    def __truediv__(self, other):
        return self * (other ** -1)

    __radd__ = __add__
    __rmul__ = __mul__

    def exp(self):
        result = math.exp(self.data)
        return Value(result, ((self, result),))

    def log(self):
        return Value(math.log(self.data), ((self, 1 / self.data),))

    def relu(self):
        return Value(max(0, self.data), ((self, float(self.data > 0)),))

    def backward(self):
        order, seen, stack = [], set(), [(self, False)]
        while stack:
            node, expanded = stack.pop()
            if expanded:
                order.append(node)
            elif node not in seen:
                seen.add(node)
                stack.append((node, True))
                stack.extend((parent, False) for parent, _ in node.parents)
        for node in order:
            node.grad = 0.0
        self.grad = 1.0
        for node in reversed(order):
            for parent, derivative in node.parents:
                parent.grad += derivative * node.grad


def softmax(values):
    peak = max(v.data for v in values)
    exps = [(v - peak).exp() for v in values]
    total = sum(exps)
    return [v / total for v in exps]


def linear(x, weights):
    return [sum(a * b for a, b in zip(x, row)) for row in weights]


def norm(x):
    mean = sum(x) / len(x)
    variance = sum((v - mean) ** 2 for v in x) / len(x)
    return [(v - mean) * (variance + 1e-5) ** -0.5 for v in x]


if __name__ == '__main__':
    x = Value(3)
    y = x * x + 2 * x
    y.backward()
    print('y =', y.data, 'dy/dx =', x.grad)
    assert y.data == 15 and x.grad == 8
model.py需要亲手编写的完整源码

先读shapes再读forward。全部参与学习，state与load保存数字。

import random
from engine import Value, linear, norm, softmax


class GPT:
    def __init__(self, vocab_size, width, context, seed=7):
        self.width, self.context = width, context
        rng = random.Random(seed)
        shapes = {
            'token': (vocab_size, width), 'position': (context, width),
            'query': (width, width), 'key': (width, width),
            'value': (width, width), 'project': (width, width),
            'expand': (2 * width, width), 'contract': (width, 2 * width),
            'head': (vocab_size, width),
        }
        self.weights = {
            name: [[Value(rng.gauss(0, 0.1)) for _ in range(cols)]
                   for _ in range(rows)] for name, (rows, cols) in shapes.items()
        }
        self.parameters = [v for matrix in self.weights.values()
                           for row in matrix for v in row]

    def forward(self, ids):
        assert 0 < len(ids) <= self.context
        w, keys, values, output = self.weights, [], [], []
        for position, token in enumerate(ids):
            x = [a + b for a, b in zip(w['token'][token], w['position'][position])]
            h = norm(x)
            q = linear(h, w['query'])
            keys.append(linear(h, w['key']))
            values.append(linear(h, w['value']))
            scores = [sum(a * b for a, b in zip(q, k)) / self.width ** 0.5
                      for k in keys]
            attention = softmax(scores)
            mixed = [sum(p * v[i] for p, v in zip(attention, values))
                     for i in range(self.width)]
            x = [a + b for a, b in zip(x, linear(mixed, w['project']))]
            hidden = [v.relu() for v in linear(norm(x), w['expand'])]
            x = [a + b for a, b in zip(x, linear(hidden, w['contract']))]
            output.append(linear(norm(x), w['head']))
        return output

    def loss(self, inputs, targets):
        logits = self.forward(inputs)
        losses = [-softmax(row)[target].log()
                  for row, target in zip(logits, targets)]
        return sum(losses) / len(losses)

    def state(self):
        return {name: [[v.data for v in row] for row in matrix]
                for name, matrix in self.weights.items()}

    def load(self, state):
        assert set(state) == set(self.weights), 'checkpoint keys'
        for name, matrix in self.weights.items():
            assert len(state[name]) == len(matrix), 'checkpoint rows'
            for row, saved in zip(matrix, state[name]):
                assert len(saved) == len(row), 'checkpoint columns'
                for value, number in zip(row, saved):
                    value.data = float(number)

检查理解

导数检查得到15.0和8.0；形状检查得到2 10。解释为何未来词元不改变先前分数。

遇到问题时

检查与zip的维度。所有嵌入、、MLP和输出权重都参与。

继续前进

请选择要取消的待付款预约。每次取消都需要确认。

你所在国家的时间:

关闭