notrito commited on
Commit
8e15344
·
verified ·
1 Parent(s): 0241fd0

Upload model.py

Browse files
Files changed (1) hide show
  1. model.py +113 -0
model.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ from torch.nn import functional as F
4
+
5
+ class Head(nn.Module):
6
+ """ one head of self-attention """
7
+
8
+ def __init__(self, n_embd, head_size, block_size, dropout):
9
+ super().__init__()
10
+ self.key = nn.Linear(n_embd, head_size, bias=False)
11
+ self.query = nn.Linear(n_embd, head_size, bias=False)
12
+ self.value = nn.Linear(n_embd, head_size, bias=False)
13
+ self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))
14
+ self.dropout = nn.Dropout(dropout)
15
+
16
+ def forward(self, x):
17
+ B,T,C = x.shape
18
+ k = self.key(x)
19
+ q = self.query(x)
20
+ wei = q @ k.transpose(-2,-1) * C**-0.5
21
+ wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf'))
22
+ wei = F.softmax(wei, dim=-1)
23
+ wei = self.dropout(wei)
24
+ v = self.value(x)
25
+ out = wei @ v
26
+ return out
27
+
28
+ class MultiHeadAttention(nn.Module):
29
+ """ multiple heads of self-attention in parallel """
30
+
31
+ def __init__(self, n_embd, num_heads, head_size, block_size, dropout):
32
+ super().__init__()
33
+ self.heads = nn.ModuleList([Head(n_embd, head_size, block_size, dropout) for _ in range(num_heads)])
34
+ self.proj = nn.Linear(n_embd, n_embd)
35
+ self.dropout = nn.Dropout(dropout)
36
+
37
+ def forward(self, x):
38
+ out = torch.cat([h(x) for h in self.heads], dim=-1)
39
+ out = self.dropout(self.proj(out))
40
+ return out
41
+
42
+ class FeedFoward(nn.Module):
43
+ """ a simple linear layer followed by a non-linearity """
44
+
45
+ def __init__(self, n_embd, dropout):
46
+ super().__init__()
47
+ self.net = nn.Sequential(
48
+ nn.Linear(n_embd, 4 * n_embd),
49
+ nn.ReLU(),
50
+ nn.Linear(4 * n_embd, n_embd),
51
+ nn.Dropout(dropout),
52
+ )
53
+
54
+ def forward(self, x):
55
+ return self.net(x)
56
+
57
+ class Block(nn.Module):
58
+ """ Transformer block: communication followed by computation """
59
+
60
+ def __init__(self, n_embd, n_head, block_size, dropout):
61
+ super().__init__()
62
+ head_size = n_embd // n_head
63
+ self.sa = MultiHeadAttention(n_embd, n_head, head_size, block_size, dropout)
64
+ self.ffwd = FeedFoward(n_embd, dropout)
65
+ self.ln1 = nn.LayerNorm(n_embd)
66
+ self.ln2 = nn.LayerNorm(n_embd)
67
+
68
+ def forward(self, x):
69
+ x = x + self.sa(self.ln1(x))
70
+ x = x + self.ffwd(self.ln2(x))
71
+ return x
72
+
73
+ class BigramLanguageModel(nn.Module):
74
+
75
+ def __init__(self, vocab_size, n_embd, block_size, n_head, n_layer, dropout):
76
+ super().__init__()
77
+ self.block_size = block_size
78
+ self.token_embedding_table = nn.Embedding(vocab_size, n_embd)
79
+ self.position_embedding_table = nn.Embedding(block_size, n_embd)
80
+ self.blocks = nn.Sequential(*[Block(n_embd, n_head, block_size, dropout) for _ in range(n_layer)])
81
+ self.ln_f = nn.LayerNorm(n_embd)
82
+ self.lm_head = nn.Linear(n_embd, vocab_size)
83
+
84
+ def forward(self, idx, targets=None):
85
+ device = idx.device
86
+ B, T = idx.shape
87
+
88
+ tok_emb = self.token_embedding_table(idx)
89
+ pos_emb = self.position_embedding_table(torch.arange(T, device=device))
90
+ x = tok_emb + pos_emb
91
+ x = self.blocks(x)
92
+ x = self.ln_f(x)
93
+ logits = self.lm_head(x)
94
+
95
+ if targets is None:
96
+ loss = None
97
+ else:
98
+ B, T, C = logits.shape
99
+ logits = logits.view(B*T, C)
100
+ targets = targets.view(B*T)
101
+ loss = F.cross_entropy(logits, targets)
102
+
103
+ return logits, loss
104
+
105
+ def generate(self, idx, max_new_tokens, temperature=1.0):
106
+ for _ in range(max_new_tokens):
107
+ idx_cond = idx[:, -self.block_size:]
108
+ logits, loss = self(idx_cond)
109
+ logits = logits[:, -1, :] / temperature
110
+ probs = F.softmax(logits, dim=-1)
111
+ idx_next = torch.multinomial(probs, num_samples=1)
112
+ idx = torch.cat((idx, idx_next), dim=1)
113
+ return idx