module Component.SigmoidWeightedGate import Std.Word import Float32Exact import BinaryFloatModel import Float32Model import FloatLiteralSpec import Component.Pointwise import Model.Word32 import Std.Natural def sigmoidWeightedGateOne : Nat = (compile-time (float32ExactRational 1 1)) -- Float32Operations uses semantic approximation selectors, not the encoded -- SM86 MUFU selector bytes. The model maps exp2 to 2 and reciprocal to 4. def sigmoidWeightedGateExp2 : Nat = 2 -- The activation commonly called SiLU or swish, expressed through the -- available exp2 and reciprocal operations. The provider determines the -- approximations and every intermediate rounding; the GPU realization must -- keep this order so the semantic and machine models agree. In particular, -- the derivative below is analytic, not a derivative of rounded f32 bits. def sigmoidFromExp2With = (lambda unrestricted operations : Float32Operations . unrestricted x : Nat . let unrestricted scaled = (f32Multiply operations (f32Negate operations x) float32Log2E) in let unrestricted exponential = (f32Approximate operations sigmoidWeightedGateExp2 scaled) in f32Approximate operations f32ApproximateReciprocal (f32Add operations sigmoidWeightedGateOne exponential)) -- The logistic function's pullback from its saved output s: (dy (1 - s)) s, -- PyTorch's sigmoid_backward order (a * (1 - b) * b). Using the output -- rather than recomputing exp keeps the forward's approximation and the -- backward's in agreement. def propagateSigmoidOutputWith = (lambda unrestricted operations : Float32Operations . unrestricted sigmoid : Nat . unrestricted upstream : Nat . f32Multiply operations (f32Multiply operations upstream (f32Add operations sigmoidWeightedGateOne (f32Negate operations sigmoid))) sigmoid) def sigmoidWeightedGateWith = (lambda unrestricted operations : Float32Operations . unrestricted x : Nat . f32Multiply operations x (sigmoidFromExp2With operations x)) def propagateSigmoidWeightedGateWith = (lambda unrestricted operations : Float32Operations . unrestricted x : Nat . unrestricted upstream : Nat . let unrestricted sigmoid = (sigmoidFromExp2With operations x) in let unrestricted complement = (f32Add operations sigmoidWeightedGateOne (f32Negate operations sigmoid)) in let unrestricted slope = (f32Multiply operations sigmoid (f32Add operations sigmoidWeightedGateOne (f32Multiply operations x complement))) in f32Multiply operations upstream slope) -- h = up * gate(x), so its two input pullbacks have different factors. def sigmoidGatedProductWith = (lambda unrestricted operations : Float32Operations . unrestricted up : Nat . unrestricted gate : Nat . f32Multiply operations up (sigmoidWeightedGateWith operations gate)) def propagateSigmoidGatedProductUpWith = (lambda unrestricted operations : Float32Operations . unrestricted gate : Nat . unrestricted upstream : Nat . f32Multiply operations upstream (sigmoidWeightedGateWith operations gate)) def propagateSigmoidGatedProductGateWith = (lambda unrestricted operations : Float32Operations . unrestricted up : Nat . unrestricted gate : Nat . unrestricted upstream : Nat . propagateSigmoidWeightedGateWith operations gate (f32Multiply operations upstream up)) -- The elementwise compiler consumes the same binary32 operation tree. These -- forms are independent of any target and can be fused with surrounding -- pointwise work. A realization must respect the explicit roundings. def sigmoidPointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . let unrestricted unary = (pointwiseUnary StdU32 inputs) in let unrestricted binary = (pointwiseBinary StdU32 inputs) in let unrestricted one = (constructor PointwiseExpression PointwiseConstant StdU32 inputs (modelWord32FromNaturalTruncated sigmoidWeightedGateOne)) in let unrestricted log2e = (constructor PointwiseExpression PointwiseConstant StdU32 inputs (modelWord32FromNaturalTruncated float32Log2E)) in then (unary PointwiseReciprocal) (binary PointwiseAdd one) (unary PointwiseExponentialBase2) (binary PointwiseMultiply (unary PointwiseNegate x)) log2e) def sigmoidWeightedPointwise = (lambda erased inputs : Nat . unrestricted x : (family PointwiseExpression StdU32 inputs) . pointwiseBinary StdU32 inputs PointwiseMultiply x (sigmoidPointwise inputs x)) def sigmoidGatedProductPointwise = (lambda erased inputs : Nat . unrestricted up : (family PointwiseExpression StdU32 inputs) . unrestricted gate : (family PointwiseExpression StdU32 inputs) . pointwiseBinary StdU32 inputs PointwiseMultiply up (sigmoidWeightedPointwise inputs gate)) def propagateSigmoidGatedProductUpPointwise = (lambda erased inputs : Nat . unrestricted gate : (family PointwiseExpression StdU32 inputs) . unrestricted upstream : (family PointwiseExpression StdU32 inputs) . pointwiseBinary StdU32 inputs PointwiseMultiply upstream (sigmoidWeightedPointwise inputs gate)) def propagateSigmoidGatedProductGatePointwise = (lambda erased inputs : Nat . unrestricted up : (family PointwiseExpression StdU32 inputs) . unrestricted gate : (family PointwiseExpression StdU32 inputs) . unrestricted upstream : (family PointwiseExpression StdU32 inputs) . let unrestricted binary = (pointwiseBinary StdU32 inputs) in let unrestricted multiply = (binary PointwiseMultiply) in let unrestricted add = (binary PointwiseAdd) in let unrestricted one = (constructor PointwiseExpression PointwiseConstant StdU32 inputs (modelWord32FromNaturalTruncated sigmoidWeightedGateOne)) in let unrestricted sigmoid = (sigmoidPointwise inputs gate) in let unrestricted complement = (add one (pointwiseUnary StdU32 inputs PointwiseNegate sigmoid)) in let unrestricted slope = (multiply sigmoid (add one (multiply gate complement))) in multiply (multiply upstream up) slope)