1module Component.CubicTanhGateSigmoid
2
3import Component.Pointwise
4import Component.SigmoidWeightedGate
5import Float32Exact
6import Float32Model
7import FloatLiteralSpec
8import Model.Word32
9import Std.Natural
10import Std.Word
11import BinaryFloatModel
12
13-- GPT-2's activation ("gelu_new"), x/2 (1 + tanh u) with
14-- u = sqrt(2/pi) (x + 0.044715 x^3), evaluated through the identity
15-- (1 + tanh u) / 2 = sigmoid(2u). The value is x sigmoid(v) with
16-- v = x (c1 + c2 x^2), c1 = 2 sqrt(2/pi), c2 = 0.044715 c1, and its
17-- pullback dy (s + x s (1 - s) (c1 + 3 c2 x^2)) with s = sigmoid(v).
18-- The sigmoid uses the exp2 and reciprocal operations of
19-- Component.SigmoidWeightedGate, so no approximate tanh enters: on SM86
20-- MUFU.TANH carries a relative error near 2^-11, while EX2 and RCP are
21-- within a few ulp. Every intermediate rounding is explicit, and the
22-- pointwise forms below keep exactly this operation order.
23--
24-- The three binary32 words are the nearest to the real constants: each
25-- rational below is within 10^-16 of its constant and at least 0.017 ulp
26-- from a rounding boundary, so its correctly rounded binary32 word is the
27-- constant's (c1 0x3fcc422a, c2 0x3d922279, 3 c2 0x3e5b33b6).
28def cubicTanhSigmoidScale : Nat =
29 (nat-add (float32ExactRational 15957691216057307 10000000000000000) 0)
30
31def cubicTanhSigmoidCubic : Nat =
32 (nat-add (float32ExactRational 713548162726002488 10000000000000000000) 0)
33
34def cubicTanhSigmoidSlopeCubic : Nat =
35 (nat-add (float32ExactRational 2140644488178007463 10000000000000000000) 0)
36
37def cubicTanhSigmoidScaleWord : (equal Nat cubicTanhSigmoidScale 0x3fcc422a) =
38 (refl Nat 0x3fcc422a)
39
40def cubicTanhSigmoidCubicWord : (equal Nat cubicTanhSigmoidCubic 0x3d922279) =
41 (refl Nat 0x3d922279)
42
43def cubicTanhSigmoidSlopeCubicWord : (equal Nat cubicTanhSigmoidSlopeCubic 0x3e5b33b6) =
44 (refl Nat 0x3e5b33b6)
45
46-- v = x (c1 + c2 x^2)
47def cubicTanhSigmoidArgumentWith =
48 (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
49 then (f32Multiply operations x)
50 (f32Add operations cubicTanhSigmoidScale)
51 (f32Multiply operations cubicTanhSigmoidCubic)
52 (f32Multiply operations x)
53 x)
54
55def cubicTanhSigmoidGateWith =
56 (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
57 f32Multiply
58 operations
59 x
60 (sigmoidFromExp2With operations (cubicTanhSigmoidArgumentWith operations x)))
61
62def propagateCubicTanhSigmoidGateWith =
63 (lambda unrestricted operations : Float32Operations .
64 unrestricted x : Nat .
65 unrestricted upstream : Nat .
66 let unrestricted s =
67 (sigmoidFromExp2With operations (cubicTanhSigmoidArgumentWith operations x))
68 in
69 let unrestricted complement =
70 (f32Add operations sigmoidWeightedGateOne (f32Negate operations s))
71 in
72 let unrestricted slope =
73 (f32Add
74 operations
75 cubicTanhSigmoidScale
76 (f32Multiply operations cubicTanhSigmoidSlopeCubic (f32Multiply operations x x)))
77 in
78 f32Multiply
79 operations
80 upstream
81 (f32Add
82 operations
83 s
84 (f32Multiply
85 operations
86 (f32Multiply operations x (f32Multiply operations s complement))
87 slope)))
88
89-- ---- the same trees as pointwise expressions ----
90def cubicTanhSigmoidConstant =
91 (lambda erased inputs : Nat . unrestricted word : Nat .
92 constructor
93 PointwiseExpression
94 PointwiseConstant
95 StdU32
96 inputs
97 (modelWord32FromNaturalTruncated word))
98
99def cubicTanhSigmoidArgumentPointwise =
100 (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
101 let unrestricted multiply =
102 (pointwiseBinary StdU32 inputs PointwiseMultiply)
103 in
104 let unrestricted add =
105 (pointwiseBinary StdU32 inputs PointwiseAdd)
106 in
107 multiply
108 x
109 (add
110 (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale)
111 (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidCubic) x x)))
112
113def cubicTanhSigmoidGatePointwise =
114 (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
115 pointwiseBinary
116 StdU32
117 inputs
118 PointwiseMultiply
119 x
120 (sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x)))
121
122def propagateCubicTanhSigmoidGatePointwise =
123 (lambda erased inputs : Nat .
124 unrestricted x : (family PointwiseExpression StdU32 inputs) .
125 unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
126 let unrestricted multiply =
127 (pointwiseBinary StdU32 inputs PointwiseMultiply)
128 in
129 let unrestricted add =
130 (pointwiseBinary StdU32 inputs PointwiseAdd)
131 in
132 let unrestricted s =
133 (sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x))
134 in
135 let unrestricted complement =
136 (add
137 (cubicTanhSigmoidConstant inputs sigmoidWeightedGateOne)
138 (pointwiseUnary StdU32 inputs PointwiseNegate s))
139 in
140 let unrestricted slope =
141 (add
142 (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale)
143 (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidSlopeCubic) x x))
144 in
145 multiply upstream (add s (multiply (nest multiply x s complement) slope)))
146
147-- propagateCubicTanhSigmoidGatePointwise with its sigmoid given: the tree
148-- names s = sigmoid(v) three times, and a compiler evaluates each node of a
149-- tree once, so the same sigmoid would be evaluated three times. Given as
150-- an operand (an input already holding the sigmoid tree's value), every
151-- node computes the same word from the same operands: the pullback's value
152-- is unchanged.
153def propagateCubicTanhSigmoidGateWithSigmoidPointwise =
154 (lambda erased inputs : Nat .
155 unrestricted x : (family PointwiseExpression StdU32 inputs) .
156 unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
157 unrestricted s : (family PointwiseExpression StdU32 inputs) .
158 let unrestricted multiply =
159 (pointwiseBinary StdU32 inputs PointwiseMultiply)
160 in
161 let unrestricted add =
162 (pointwiseBinary StdU32 inputs PointwiseAdd)
163 in
164 let unrestricted complement =
165 (add
166 (cubicTanhSigmoidConstant inputs sigmoidWeightedGateOne)
167 (pointwiseUnary StdU32 inputs PointwiseNegate s))
168 in
169 let unrestricted slope =
170 (add
171 (cubicTanhSigmoidConstant inputs cubicTanhSigmoidScale)
172 (nest multiply (cubicTanhSigmoidConstant inputs cubicTanhSigmoidSlopeCubic) x x))
173 in
174 multiply upstream (add s (multiply (nest multiply x s complement) slope)))
175
176-- the sigmoid the pullback names, of x
177def cubicTanhSigmoidOfPointwise =
178 (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
179 sigmoidPointwise inputs (cubicTanhSigmoidArgumentPointwise inputs x))The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.