Source/Packages

Component.SigmoidWeightedGate

packages/components/src/Component/SigmoidWeightedGate.alpha

181 lines14 declarations6.6 KiBSHA-256 7302f612c9f7

Complete file · line 99

SigmoidWeightedGate.alpha

Definition view
1module Component.SigmoidWeightedGate
2
3import Std.Word
4import Float32Exact
5import BinaryFloatModel
6import Float32Model
7import FloatLiteralSpec
8import Component.Pointwise
9import Model.Word32
10import Std.Natural
11
12def sigmoidWeightedGateOne : Nat =
13  (compile-time (float32ExactRational 1 1))
14
15-- Float32Operations uses semantic approximation selectors, not the encoded
16-- SM86 MUFU selector bytes. The model maps exp2 to 2 and reciprocal to 4.
17def sigmoidWeightedGateExp2 : Nat =
18  2
19
20-- The activation commonly called SiLU or swish, expressed through the
21-- available exp2 and reciprocal operations. The provider determines the
22-- approximations and every intermediate rounding; the GPU realization must
23-- keep this order so the semantic and machine models agree. In particular,
24-- the derivative below is analytic, not a derivative of rounded f32 bits.
25def sigmoidFromExp2With =
26  (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
27    let unrestricted scaled =
28      (f32Multiply operations (f32Negate operations x) float32Log2E)
29      in
30      let unrestricted exponential =
31        (f32Approximate operations sigmoidWeightedGateExp2 scaled)
32        in
33        f32Approximate
34          operations
35          f32ApproximateReciprocal
36          (f32Add operations sigmoidWeightedGateOne exponential))
37
38-- The logistic function's pullback from its saved output s: (dy (1 - s)) s,
39-- PyTorch's sigmoid_backward order (a * (1 - b) * b). Using the output
40-- rather than recomputing exp keeps the forward's approximation and the
41-- backward's in agreement.
42def propagateSigmoidOutputWith =
43  (lambda unrestricted operations : Float32Operations .
44    unrestricted sigmoid : Nat .
45    unrestricted upstream : Nat .
46    f32Multiply
47      operations
48      (f32Multiply
49        operations
50        upstream
51        (f32Add operations sigmoidWeightedGateOne (f32Negate operations sigmoid)))
52      sigmoid)
53
54def sigmoidWeightedGateWith =
55  (lambda unrestricted operations : Float32Operations . unrestricted x : Nat .
56    f32Multiply operations x (sigmoidFromExp2With operations x))
57
58def propagateSigmoidWeightedGateWith =
59  (lambda unrestricted operations : Float32Operations .
60    unrestricted x : Nat .
61    unrestricted upstream : Nat .
62    let unrestricted sigmoid =
63      (sigmoidFromExp2With operations x)
64      in
65      let unrestricted complement =
66        (f32Add operations sigmoidWeightedGateOne (f32Negate operations sigmoid))
67        in
68        let unrestricted slope =
69          (f32Multiply
70            operations
71            sigmoid
72            (f32Add operations sigmoidWeightedGateOne (f32Multiply operations x complement)))
73          in
74          f32Multiply operations upstream slope)
75
76-- h = up * gate(x), so its two input pullbacks have different factors.
77def sigmoidGatedProductWith =
78  (lambda unrestricted operations : Float32Operations .
79    unrestricted up : Nat .
80    unrestricted gate : Nat .
81    f32Multiply operations up (sigmoidWeightedGateWith operations gate))
82
83def propagateSigmoidGatedProductUpWith =
84  (lambda unrestricted operations : Float32Operations .
85    unrestricted gate : Nat .
86    unrestricted upstream : Nat .
87    f32Multiply operations upstream (sigmoidWeightedGateWith operations gate))
88
89def propagateSigmoidGatedProductGateWith =
90  (lambda unrestricted operations : Float32Operations .
91    unrestricted up : Nat .
92    unrestricted gate : Nat .
93    unrestricted upstream : Nat .
94    propagateSigmoidWeightedGateWith operations gate (f32Multiply operations upstream up))
95
96-- The elementwise compiler consumes the same binary32 operation tree. These
97-- forms are independent of any target and can be fused with surrounding
98-- pointwise work. A realization must respect the explicit roundings.
99def sigmoidPointwise =
100  (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
101    let unrestricted unary =
102      (pointwiseUnary StdU32 inputs)
103      in
104      let unrestricted binary =
105        (pointwiseBinary StdU32 inputs)
106        in
107        let unrestricted one =
108          (constructor
109            PointwiseExpression
110            PointwiseConstant
111            StdU32
112            inputs
113            (modelWord32FromNaturalTruncated sigmoidWeightedGateOne))
114          in
115          let unrestricted log2e =
116            (constructor
117              PointwiseExpression
118              PointwiseConstant
119              StdU32
120              inputs
121              (modelWord32FromNaturalTruncated float32Log2E))
122            in
123            then (unary PointwiseReciprocal)
124              (binary PointwiseAdd one)
125              (unary PointwiseExponentialBase2)
126              (binary PointwiseMultiply (unary PointwiseNegate x))
127              log2e)
128
129def sigmoidWeightedPointwise =
130  (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) .
131    pointwiseBinary StdU32 inputs PointwiseMultiply x (sigmoidPointwise inputs x))
132
133def sigmoidGatedProductPointwise =
134  (lambda erased inputs : Nat .
135    unrestricted up : (family PointwiseExpression StdU32 inputs) .
136    unrestricted gate : (family PointwiseExpression StdU32 inputs) .
137    pointwiseBinary StdU32 inputs PointwiseMultiply up (sigmoidWeightedPointwise inputs gate))
138
139def propagateSigmoidGatedProductUpPointwise =
140  (lambda erased inputs : Nat .
141    unrestricted gate : (family PointwiseExpression StdU32 inputs) .
142    unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
143    pointwiseBinary
144      StdU32
145      inputs
146      PointwiseMultiply
147      upstream
148      (sigmoidWeightedPointwise inputs gate))
149
150def propagateSigmoidGatedProductGatePointwise =
151  (lambda erased inputs : Nat .
152    unrestricted up : (family PointwiseExpression StdU32 inputs) .
153    unrestricted gate : (family PointwiseExpression StdU32 inputs) .
154    unrestricted upstream : (family PointwiseExpression StdU32 inputs) .
155    let unrestricted binary =
156      (pointwiseBinary StdU32 inputs)
157      in
158      let unrestricted multiply =
159        (binary PointwiseMultiply)
160        in
161        let unrestricted add =
162          (binary PointwiseAdd)
163          in
164          let unrestricted one =
165            (constructor
166              PointwiseExpression
167              PointwiseConstant
168              StdU32
169              inputs
170              (modelWord32FromNaturalTruncated sigmoidWeightedGateOne))
171            in
172            let unrestricted sigmoid =
173              (sigmoidPointwise inputs gate)
174              in
175              let unrestricted complement =
176                (add one (pointwiseUnary StdU32 inputs PointwiseNegate sigmoid))
177                in
178                let unrestricted slope =
179                  (multiply sigmoid (add one (multiply gate complement)))
180                  in
181                  multiply (multiply upstream up) slope)

The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.