module Component.CubicTanhGateSigmoid import Component.Pointwise import Component.SigmoidWeightedGate import Float32Exact import Float32Model import FloatLiteralSpec import Model.Word32 import Std.Natural import Std.Word import BinaryFloatModel -- GPT-2's activation ("gelu_new"), x/2 (1 + tanh u) with -- u = sqrt(2/pi) (x + 0.044715 x^3), evaluated through the identity -- (1 + tanh u) / 2 = sigmoid(2u). The value is x sigmoid(v) with -- v = x (c1 + c2 x^2), c1 = 2 sqrt(2/pi), c2 = 0.044715 c1, and its -- pullback dy (s + x s (1 - s) (c1 + 3 c2 x^2)) with s = sigmoid(v). -- The sigmoid uses the exp2 and reciprocal operations of -- Component.SigmoidWeightedGate, so no approximate tanh enters: on SM86 -- MUFU.TANH carries a relative error near 2^-11, while EX2 and RCP are -- within a few ulp. Every intermediate rounding is explicit, and the -- pointwise forms below keep exactly this operation order. -- -- The three binary32 words are the nearest to the real constants: each -- rational below is within 10^-16 of its constant and at least 0.017 ulp -- from a rounding boundary, so its correctly rounded binary32 word is the -- constant's (c1 0x3fcc422a, c2 0x3d922279, 3 c2 0x3e5b33b6). def cubicTanhSigmoidScale : Nat = (nat-add (float32ExactRational 15957691216057307 10000000000000000) 0) def cubicTanhSigmoidCubic : Nat = (nat-add (float32ExactRational 713548162726002488 10000000000000000000) 0) def cubicTanhSigmoidSlopeCubic : Nat = (nat-add (float32ExactRational 2140644488178007463 10000000000000000000) 0) def cubicTanhSigmoidScaleWord : (equal Nat cubicTanhSigmoidScale 0x3fcc422a) = (refl Nat 0x3fcc422a) def cubicTanhSigmoidCubicWord : (equal Nat cubicTanhSigmoidCubic 0x3d922279) = (refl Nat 0x3d922279) def cubicTanhSigmoidSlopeCubicWord : (equal Nat cubicTanhSigmoidSlopeCubic 0x3e5b33b6) = (refl Nat 0x3e5b33b6) -- v = x (c1 + c2 x^2) def cubicTanhSigmoidArgumentWith = (lambda unrestricted operations : Float32Operations . unrestricted x : Nat . then (f32Multiply operations x) (f32Add operations cubicTanhSigmoidScale) (f32Multiply operations cubicTanhSigmoidCubic) (f32Multiply operations x) x) def cubicTanhSigmoidGateWith = (lambda unrestricted operations : Float32Operations . unrestricted x : Nat . f32Multiply operations x (sigmoidFromExp2With operations (cubicTanhSigmoidArgumentWith operations x))) def propagateCubicTanhSigmoidGateWith = (lambda unrestricted operations : Float32Operations . unrestricted x : Nat . unrestricted upstream : Nat . let unrestricted s = (sigmoidFromExp2With operations (cubicTanhSigmoidArgumentWith operations x)) in let unrestricted complement = (f32Add operations sigmoidWeightedGateOne (f32Negate operations s)) in let unrestricted slope = (f32Add operations cubicTanhSigmoidScale (f32Multiply operations cubicTanhSigmoidSlopeCubic (f32Multiply operations x x))) in f32Multiply operations upstream (f32Add operations s (f32Multiply operations (f32Multiply operations x (f32Multiply operations s complement)) slope))) -- ---- the same trees as pointwise expressions ---- def cubicTanhSigmoidConstant = (lambda erased inputs : Nat . unrestricted word : Nat . constructor PointwiseExpression PointwiseConstant StdU32 inputs (modelWord32FromNaturalTruncated word)) def cubicTanhSigmoidArgumentPointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . let unrestricted multiply = (pointwiseBinary StdU32 inputs PointwiseMultiply) in let unrestricted add = (pointwiseBinary StdU32 inputs PointwiseAdd) in multiply x (add (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale) (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidCubic) x x))) def cubicTanhSigmoidGatePointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . pointwiseBinary StdU32 inputs PointwiseMultiply x (sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x))) def propagateCubicTanhSigmoidGatePointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . unrestricted upstream : (family PointwiseExpression StdU32 inputs) . let unrestricted multiply = (pointwiseBinary StdU32 inputs PointwiseMultiply) in let unrestricted add = (pointwiseBinary StdU32 inputs PointwiseAdd) in let unrestricted s = (sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x)) in let unrestricted complement = (add (cubicTanhSigmoidConstant inputs sigmoidWeightedGateOne) (pointwiseUnary StdU32 inputs PointwiseNegate s)) in let unrestricted slope = (add (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale) (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidSlopeCubic) x x)) in multiply upstream (add s (multiply (nest multiply x s complement) slope))) -- propagateCubicTanhSigmoidGatePointwise with its sigmoid given: the tree -- names s = sigmoid(v) three times, and a compiler evaluates each node of a -- tree once, so the same sigmoid would be evaluated three times. Given as -- an operand (an input already holding the sigmoid tree's value), every -- node computes the same word from the same operands: the pullback's value -- is unchanged. def propagateCubicTanhSigmoidGateWithSigmoidPointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . unrestricted upstream : (family PointwiseExpression StdU32 inputs) . unrestricted s : (family PointwiseExpression StdU32 inputs) . let unrestricted multiply = (pointwiseBinary StdU32 inputs PointwiseMultiply) in let unrestricted add = (pointwiseBinary StdU32 inputs PointwiseAdd) in let unrestricted complement = (add (cubicTanhSigmoidConstant inputs sigmoidWeightedGateOne) (pointwiseUnary StdU32 inputs PointwiseNegate s)) in let unrestricted slope = (add (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale) (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidSlopeCubic) x x)) in multiply upstream (add s (multiply (nest multiply x s complement) slope))) -- the sigmoid the pullback names, of x def cubicTanhSigmoidOfPointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x))