Source/Systems

GPT2.Model

systems/gpt2/src/GPT2/Model.alpha

75 lines16 declarations3.3 KiBSHA-256 de41e63ea477

Complete file · line 75

Model.alpha

Definition view
1module GPT2.Model
2
3import Component.CubicTanhGateSigmoid
4import Float32Exact
5import FloatLiteralSpec
6import Model.LearnedPositionDecoder
7import Std.Natural
8import BinaryFloatModel
9
10-- GPT-2 small, openai-community/gpt2 config.json (Radford et al. 2019):
11-- vocabulary 50257 (byte-level BPE, <|endoftext|> = 50256), context 1024,
12-- 12 blocks of width 768, 12 heads of 64, feed-forward width 3072
13-- (n_inner = 4 n_embd), gelu_new, LayerNorm epsilon 1e-5, tied input and
14-- output embeddings, learned positions, biases on every linear map.
15-- The weights start fresh; the released checkpoint is not used.
16--
17-- The stored vocabulary is 50432 = 197 x 256, the multiple of 256 the exact
18-- objective's row kernels need. The 175 padded classes are never inputs or
19-- targets and their logits are filled with -1e30 before the objective
20-- (Realization.Nvidia.SM86.ColumnFillSM86), so the model is GPT-2's.
21def gpt2TokenVocabulary : Nat = 50257
22def gpt2Geometry : LearnedPositionDecoderGeometry =
23  (constructor LearnedPositionDecoderGeometry LearnedPositionDecoderGeometryValue
24    50432 gpt2TokenVocabulary 1024 12 768 12 64 3072)
25
26def gpt2ParameterCount : Nat =
27  (learnedPositionDecoderParameterCount gpt2Geometry)
28
29def gpt2EndOfTextToken : Nat = 50256
30
31-- GPT-2 initializes every weight from N(0, 0.02^2), the residual output
32-- projections (attention output and feed reduction, c_proj) from
33-- N(0, (0.02 / sqrt(2 layers))^2), every bias at zero and every LayerNorm's
34-- gain at one and shift at zero. The device normal image uses twelve
35-- uniform bytes centred at zero (variance 65535): its scale is the
36-- deviation divided by sqrt(65535). This is a CLT approximation to a
37-- Gaussian, not PyTorch's RNG stream.
38def gpt2InitializationDeviationDenominator : Nat = 50
39def gpt2InitializationScaleWord : Nat =
40  (pipe gpt2InitializationDeviationDenominator
41    (naturalMultiply gpt2InitializationDeviationDenominator)
42    (naturalMultiply 65535)
43    float32ExactInverseRoot
44    compile-time)
45-- 0.02 / sqrt(24) / sqrt(65535) = 1 / sqrt(2500 * 24 * 65535)
46def gpt2ResidualInitializationScaleWord : Nat =
47  (pipe (nest naturalMultiply
48          (naturalMultiply
49            gpt2InitializationDeviationDenominator
50            gpt2InitializationDeviationDenominator)
51          2 (learnedPositionDecoderLayersOf gpt2Geometry))
52    (naturalMultiply 65535)
53    float32ExactInverseRoot
54    compile-time)
55
56def gpt2LayerNormEpsilonNumerator : Nat = 1
57def gpt2LayerNormEpsilonDenominator : Nat = 100000
58def gpt2LayerNormEpsilonWord : Nat =
59  (nat-add (float32ExactRational gpt2LayerNormEpsilonNumerator
60    gpt2LayerNormEpsilonDenominator) 0)
61def gpt2InverseWidthWord : Nat =
62  (nat-add (float32ExactRational 1 (learnedPositionDecoderWidthOf gpt2Geometry)) 0)
63def gpt2LossInverseRowsWord : Nat =
64  (nat-add (float32ExactRational 1 (learnedPositionDecoderContextOf gpt2Geometry)) 0)
65def gpt2AttentionScaleWord : Nat =
66  (compile-time
67    (float32ExactInverseRoot (learnedPositionDecoderHeadWidthOf gpt2Geometry)))
68def gpt2AttentionBaseTwoScaleWord : Nat =
69  (compile-time
70    (float32ExactInverseRootNaturalLogTwo
71      (learnedPositionDecoderHeadWidthOf gpt2Geometry)))
72
73-- The feed-forward activation: gelu_new through the sigmoid identity.
74def gpt2FeedForwardWith = cubicTanhSigmoidGateWith
75def gpt2FeedForwardPullbackWith = propagateCubicTanhSigmoidGateWith

The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.