1module Component.SigmoidWeightedGate
2
3import Std.Word
4import Float32Exact
5import BinaryFloatModel
6import Float32Model
7import FloatLiteralSpec
8import Component.Pointwise
9import Model.Word32
10import Std.Natural
11
12def sigmoidWeightedGateOne : Nat =
13 (compile-time (float32ExactRational 1 1))
14
15-- Float32Operations uses semantic approximation selectors, not the encoded
16-- SM86 MUFU selector bytes. The model maps exp2 to 2 and reciprocal to 4.
17def sigmoidWeightedGateExp2 : Nat =
18 2
19
20-- The activation commonly called SiLU or swish, expressed through the
21-- available exp2 and reciprocal operations. The provider determines the
22-- approximations and every intermediate rounding; the GPU realization must
23-- keep this order so the semantic and machine models agree. In particular,
24-- the derivative below is analytic, not a derivative of rounded f32 bits.
25def sigmoidFromExp2With =
26 (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
27 let unrestricted scaled =
28 (f32Multiply operations (f32Negate operations x) float32Log2E)
29 in
30 let unrestricted exponential =
31 (f32Approximate operations sigmoidWeightedGateExp2 scaled)
32 in
33 f32Approximate
34 operations
35 f32ApproximateReciprocal
36 (f32Add operations sigmoidWeightedGateOne exponential))
37
38-- The logistic function's pullback from its saved output s: (dy (1 - s)) s,
39-- PyTorch's sigmoid_backward order (a * (1 - b) * b). Using the output
40-- rather than recomputing exp keeps the forward's approximation and the
41-- backward's in agreement.
42def propagateSigmoidOutputWith =
43 (lambda unrestricted operations : Float32Operations .
44 unrestricted sigmoid : Nat .
45 unrestricted upstream : Nat .
46 f32Multiply
47 operations
48 (f32Multiply
49 operations
50 upstream
51 (f32Add operations sigmoidWeightedGateOne (f32Negate operations sigmoid)))
52 sigmoid)
53
54def sigmoidWeightedGateWith =
55 (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
56 f32Multiply operations x (sigmoidFromExp2With operations x))
57
58def propagateSigmoidWeightedGateWith =
59 (lambda unrestricted operations : Float32Operations .
60 unrestricted x : Nat .
61 unrestricted upstream : Nat .
62 let unrestricted sigmoid =
63 (sigmoidFromExp2With operations x)
64 in
65 let unrestricted complement =
66 (f32Add operations sigmoidWeightedGateOne (f32Negate operations sigmoid))
67 in
68 let unrestricted slope =
69 (f32Multiply
70 operations
71 sigmoid
72 (f32Add operations sigmoidWeightedGateOne (f32Multiply operations x complement)))
73 in
74 f32Multiply operations upstream slope)
75
76-- h = up * gate(x), so its two input pullbacks have different factors.
77def sigmoidGatedProductWith =
78 (lambda unrestricted operations : Float32Operations .
79 unrestricted up : Nat .
80 unrestricted gate : Nat .
81 f32Multiply operations up (sigmoidWeightedGateWith operations gate))
82
83def propagateSigmoidGatedProductUpWith =
84 (lambda unrestricted operations : Float32Operations .
85 unrestricted gate : Nat .
86 unrestricted upstream : Nat .
87 f32Multiply operations upstream (sigmoidWeightedGateWith operations gate))
88
89def propagateSigmoidGatedProductGateWith =
90 (lambda unrestricted operations : Float32Operations .
91 unrestricted up : Nat .
92 unrestricted gate : Nat .
93 unrestricted upstream : Nat .
94 propagateSigmoidWeightedGateWith operations gate (f32Multiply operations upstream up))
95
96-- The elementwise compiler consumes the same binary32 operation tree. These
97-- forms are independent of any target and can be fused with surrounding
98-- pointwise work. A realization must respect the explicit roundings.
99def sigmoidPointwise =
100 (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
101 let unrestricted unary =
102 (pointwiseUnary StdU32 inputs)
103 in
104 let unrestricted binary =
105 (pointwiseBinary StdU32 inputs)
106 in
107 let unrestricted one =
108 (constructor
109 PointwiseExpression
110 PointwiseConstant
111 StdU32
112 inputs
113 (modelWord32FromNaturalTruncated sigmoidWeightedGateOne))
114 in
115 let unrestricted log2e =
116 (constructor
117 PointwiseExpression
118 PointwiseConstant
119 StdU32
120 inputs
121 (modelWord32FromNaturalTruncated float32Log2E))
122 in
123 then (unary PointwiseReciprocal)
124 (binary PointwiseAdd one)
125 (unary PointwiseExponentialBase2)
126 (binary PointwiseMultiply (unary PointwiseNegate x))
127 log2e)
128
129def sigmoidWeightedPointwise =
130 (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
131 pointwiseBinary StdU32 inputs PointwiseMultiply x (sigmoidPointwise inputs x))
132
133def sigmoidGatedProductPointwise =
134 (lambda erased inputs : Nat .
135 unrestricted up : (family PointwiseExpression StdU32 inputs) .
136 unrestricted gate : (family PointwiseExpression StdU32 inputs) .
137 pointwiseBinary StdU32 inputs PointwiseMultiply up (sigmoidWeightedPointwise inputs gate))
138
139def propagateSigmoidGatedProductUpPointwise =
140 (lambda erased inputs : Nat .
141 unrestricted gate : (family PointwiseExpression StdU32 inputs) .
142 unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
143 pointwiseBinary
144 StdU32
145 inputs
146 PointwiseMultiply
147 upstream
148 (sigmoidWeightedPointwise inputs gate))
149
150def propagateSigmoidGatedProductGatePointwise =
151 (lambda erased inputs : Nat .
152 unrestricted up : (family PointwiseExpression StdU32 inputs) .
153 unrestricted gate : (family PointwiseExpression StdU32 inputs) .
154 unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
155 let unrestricted binary =
156 (pointwiseBinary StdU32 inputs)
157 in
158 let unrestricted multiply =
159 (binary PointwiseMultiply)
160 in
161 let unrestricted add =
162 (binary PointwiseAdd)
163 in
164 let unrestricted one =
165 (constructor
166 PointwiseExpression
167 PointwiseConstant
168 StdU32
169 inputs
170 (modelWord32FromNaturalTruncated sigmoidWeightedGateOne))
171 in
172 let unrestricted sigmoid =
173 (sigmoidPointwise inputs gate)
174 in
175 let unrestricted complement =
176 (add one (pointwiseUnary StdU32 inputs PointwiseNegate sigmoid))
177 in
178 let unrestricted slope =
179 (multiply sigmoid (add one (multiply gate complement)))
180 in
181 multiply (multiply upstream up) slope)The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.