Source/Packages

Component.CubicTanhGateSigmoid

packages/components/src/Component/CubicTanhGateSigmoid.alpha

179 lines15 declarations6.9 KiBSHA-256 d73f403dc6e0

Complete file · line 177

CubicTanhGateSigmoid.alpha

Definition view
1module Component.CubicTanhGateSigmoid
2
3import Component.Pointwise
4import Component.SigmoidWeightedGate
5import Float32Exact
6import Float32Model
7import FloatLiteralSpec
8import Model.Word32
9import Std.Natural
10import Std.Word
11import BinaryFloatModel
12
13-- GPT-2's activation ("gelu_new"), x/2 (1 + tanh u) with
14-- u = sqrt(2/pi) (x + 0.044715 x^3), evaluated through the identity
15-- (1 + tanh u) / 2 = sigmoid(2u). The value is x sigmoid(v) with
16-- v = x (c1 + c2 x^2), c1 = 2 sqrt(2/pi), c2 = 0.044715 c1, and its
17-- pullback dy (s + x s (1 - s) (c1 + 3 c2 x^2)) with s = sigmoid(v).
18-- The sigmoid uses the exp2 and reciprocal operations of
19-- Component.SigmoidWeightedGate, so no approximate tanh enters: on SM86
20-- MUFU.TANH carries a relative error near 2^-11, while EX2 and RCP are
21-- within a few ulp. Every intermediate rounding is explicit, and the
22-- pointwise forms below keep exactly this operation order.
23--
24-- The three binary32 words are the nearest to the real constants: each
25-- rational below is within 10^-16 of its constant and at least 0.017 ulp
26-- from a rounding boundary, so its correctly rounded binary32 word is the
27-- constant's (c1 0x3fcc422a, c2 0x3d922279, 3 c2 0x3e5b33b6).
28def cubicTanhSigmoidScale : Nat =
29  (nat-add (float32ExactRational 15957691216057307 10000000000000000) 0)
30
31def cubicTanhSigmoidCubic : Nat =
32  (nat-add (float32ExactRational 713548162726002488 10000000000000000000) 0)
33
34def cubicTanhSigmoidSlopeCubic : Nat =
35  (nat-add (float32ExactRational 2140644488178007463 10000000000000000000) 0)
36
37def cubicTanhSigmoidScaleWord : (equal Nat cubicTanhSigmoidScale 0x3fcc422a) =
38  (refl Nat 0x3fcc422a)
39
40def cubicTanhSigmoidCubicWord : (equal Nat cubicTanhSigmoidCubic 0x3d922279) =
41  (refl Nat 0x3d922279)
42
43def cubicTanhSigmoidSlopeCubicWord : (equal Nat cubicTanhSigmoidSlopeCubic 0x3e5b33b6) =
44  (refl Nat 0x3e5b33b6)
45
46-- v = x (c1 + c2 x^2)
47def cubicTanhSigmoidArgumentWith =
48  (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
49    then (f32Multiply operations x)
50      (f32Add operations cubicTanhSigmoidScale)
51      (f32Multiply operations cubicTanhSigmoidCubic)
52      (f32Multiply operations x)
53      x)
54
55def cubicTanhSigmoidGateWith =
56  (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
57    f32Multiply
58      operations
59      x
60      (sigmoidFromExp2With operations (cubicTanhSigmoidArgumentWith operations x)))
61
62def propagateCubicTanhSigmoidGateWith =
63  (lambda unrestricted operations : Float32Operations .
64    unrestricted x : Nat .
65    unrestricted upstream : Nat .
66    let unrestricted s =
67      (sigmoidFromExp2With operations (cubicTanhSigmoidArgumentWith operations x))
68      in
69      let unrestricted complement =
70        (f32Add operations sigmoidWeightedGateOne (f32Negate operations s))
71        in
72        let unrestricted slope =
73          (f32Add
74            operations
75            cubicTanhSigmoidScale
76            (f32Multiply operations cubicTanhSigmoidSlopeCubic (f32Multiply operations x x)))
77          in
78          f32Multiply
79            operations
80            upstream
81            (f32Add
82              operations
83              s
84              (f32Multiply
85                operations
86                (f32Multiply operations x (f32Multiply operations s complement))
87                slope)))
88
89-- ---- the same trees as pointwise expressions ----
90def cubicTanhSigmoidConstant =
91  (lambda erased inputs : Nat . unrestricted word : Nat .
92    constructor
93      PointwiseExpression
94      PointwiseConstant
95      StdU32
96      inputs
97      (modelWord32FromNaturalTruncated word))
98
99def cubicTanhSigmoidArgumentPointwise =
100  (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
101    let unrestricted multiply =
102      (pointwiseBinary StdU32 inputs PointwiseMultiply)
103      in
104      let unrestricted add =
105        (pointwiseBinary StdU32 inputs PointwiseAdd)
106        in
107        multiply
108          x
109          (add
110            (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale)
111            (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidCubic) x x)))
112
113def cubicTanhSigmoidGatePointwise =
114  (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
115    pointwiseBinary
116      StdU32
117      inputs
118      PointwiseMultiply
119      x
120      (sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x)))
121
122def propagateCubicTanhSigmoidGatePointwise =
123  (lambda erased inputs : Nat .
124    unrestricted x : (family PointwiseExpression StdU32 inputs) .
125    unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
126    let unrestricted multiply =
127      (pointwiseBinary StdU32 inputs PointwiseMultiply)
128      in
129      let unrestricted add =
130        (pointwiseBinary StdU32 inputs PointwiseAdd)
131        in
132        let unrestricted s =
133          (sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x))
134          in
135          let unrestricted complement =
136            (add
137              (cubicTanhSigmoidConstant inputs sigmoidWeightedGateOne)
138              (pointwiseUnary StdU32 inputs PointwiseNegate s))
139            in
140            let unrestricted slope =
141              (add
142                (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale)
143                (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidSlopeCubic) x x))
144              in
145              multiply upstream (add s (multiply (nest multiply x s complement) slope)))
146
147-- propagateCubicTanhSigmoidGatePointwise with its sigmoid given: the tree
148-- names s = sigmoid(v) three times, and a compiler evaluates each node of a
149-- tree once, so the same sigmoid would be evaluated three times. Given as
150-- an operand (an input already holding the sigmoid tree's value), every
151-- node computes the same word from the same operands: the pullback's value
152-- is unchanged.
153def propagateCubicTanhSigmoidGateWithSigmoidPointwise =
154  (lambda erased inputs : Nat .
155    unrestricted x : (family PointwiseExpression StdU32 inputs) .
156    unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
157    unrestricted s : (family PointwiseExpression StdU32 inputs) .
158    let unrestricted multiply =
159      (pointwiseBinary StdU32 inputs PointwiseMultiply)
160      in
161      let unrestricted add =
162        (pointwiseBinary StdU32 inputs PointwiseAdd)
163        in
164        let unrestricted complement =
165          (add
166            (cubicTanhSigmoidConstant inputs sigmoidWeightedGateOne)
167            (pointwiseUnary StdU32 inputs PointwiseNegate s))
168          in
169          let unrestricted slope =
170            (add
171              (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale)
172              (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidSlopeCubic) x x))
173            in
174            multiply upstream (add s (multiply (nest multiply x s complement) slope)))
175
176-- the sigmoid the pullback names, of x
177def cubicTanhSigmoidOfPointwise =
178  (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
179    sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x))

The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.