module GPT2.Model import Component.CubicTanhGateSigmoid import Float32Exact import FloatLiteralSpec import Model.LearnedPositionDecoder import Std.Natural import BinaryFloatModel -- GPT-2 small, openai-community/gpt2 config.json (Radford et al. 2019): -- vocabulary 50257 (byte-level BPE, <|endoftext|> = 50256), context 1024, -- 12 blocks of width 768, 12 heads of 64, feed-forward width 3072 -- (n_inner = 4 n_embd), gelu_new, LayerNorm epsilon 1e-5, tied input and -- output embeddings, learned positions, biases on every linear map. -- The weights start fresh; the released checkpoint is not used. -- -- The stored vocabulary is 50432 = 197 x 256, the multiple of 256 the exact -- objective's row kernels need. The 175 padded classes are never inputs or -- targets and their logits are filled with -1e30 before the objective -- (Realization.Nvidia.SM86.ColumnFillSM86), so the model is GPT-2's. def gpt2TokenVocabulary : Nat = 50257 def gpt2Geometry : LearnedPositionDecoderGeometry = (constructor LearnedPositionDecoderGeometry LearnedPositionDecoderGeometryValue 50432 gpt2TokenVocabulary 1024 12 768 12 64 3072) def gpt2ParameterCount : Nat = (learnedPositionDecoderParameterCount gpt2Geometry) def gpt2EndOfTextToken : Nat = 50256 -- GPT-2 initializes every weight from N(0, 0.02^2), the residual output -- projections (attention output and feed reduction, c_proj) from -- N(0, (0.02 / sqrt(2 layers))^2), every bias at zero and every LayerNorm's -- gain at one and shift at zero. The device normal image uses twelve -- uniform bytes centred at zero (variance 65535): its scale is the -- deviation divided by sqrt(65535). This is a CLT approximation to a -- Gaussian, not PyTorch's RNG stream. def gpt2InitializationDeviationDenominator : Nat = 50 def gpt2InitializationScaleWord : Nat = (pipe gpt2InitializationDeviationDenominator (naturalMultiply gpt2InitializationDeviationDenominator) (naturalMultiply 65535) float32ExactInverseRoot compile-time) -- 0.02 / sqrt(24) / sqrt(65535) = 1 / sqrt(2500 * 24 * 65535) def gpt2ResidualInitializationScaleWord : Nat = (pipe (nest naturalMultiply (naturalMultiply gpt2InitializationDeviationDenominator gpt2InitializationDeviationDenominator) 2 (learnedPositionDecoderLayersOf gpt2Geometry)) (naturalMultiply 65535) float32ExactInverseRoot compile-time) def gpt2LayerNormEpsilonNumerator : Nat = 1 def gpt2LayerNormEpsilonDenominator : Nat = 100000 def gpt2LayerNormEpsilonWord : Nat = (nat-add (float32ExactRational gpt2LayerNormEpsilonNumerator gpt2LayerNormEpsilonDenominator) 0) def gpt2InverseWidthWord : Nat = (nat-add (float32ExactRational 1 (learnedPositionDecoderWidthOf gpt2Geometry)) 0) def gpt2LossInverseRowsWord : Nat = (nat-add (float32ExactRational 1 (learnedPositionDecoderContextOf gpt2Geometry)) 0) def gpt2AttentionScaleWord : Nat = (compile-time (float32ExactInverseRoot (learnedPositionDecoderHeadWidthOf gpt2Geometry))) def gpt2AttentionBaseTwoScaleWord : Nat = (compile-time (float32ExactInverseRootNaturalLogTwo (learnedPositionDecoderHeadWidthOf gpt2Geometry))) -- The feed-forward activation: gelu_new through the sigmoid identity. def gpt2FeedForwardWith = cubicTanhSigmoidGateWith def gpt2FeedForwardPullbackWith = propagateCubicTanhSigmoidGateWith