Source/Packages

Component.Pointwise

packages/components/src/Component/Pointwise.alpha

408 lines49 declarations15.8 KiBSHA-256 b4c565ce95d3

Complete file · line 299

Pointwise.alpha

Definition view
1module Component.Pointwise
2
3import Data.BFloat16
4import BinaryFloatModel
5import Float32Model
6import FloatLiteralSpec
7import Model.Config
8import Model.Parameter
9import Model.Word32
10import Std.Fin
11import Std.List
12import Std.Natural
13import Std.Word
14
15-- A scalar expression applied independently at each tensor element. The word
16-- type binds literal/input representation; a binary64 backend cannot consume
17-- a binary32 expression by reinterpreting its literals. Input indices are
18-- bounded in the type; hardware, pointers and thread counts do not occur here.
19-- The chosen arithmetic provider owns rounding. An explicit rounding operation
20-- remains a boundary that a backend may not fuse across.
21family PointwiseUnary : Type 0
22constructor PointwiseNegate
23constructor PointwiseSquare
24constructor PointwiseRoundBF16
25constructor PointwiseReciprocal
26constructor PointwiseReciprocalSquareRoot
27constructor PointwiseSquareRoot
28constructor PointwiseTanh
29constructor PointwiseExponentialBase2
30
31end-family
32
33family PointwiseBinary : Type 0
34constructor PointwiseAdd
35constructor PointwiseMultiply
36constructor PointwiseSubtract
37-- threshold(x, other): zero when x <= 0, otherwise other. NaNs take the
38-- other arm. This is the upstream ReLU/threshold-backward rule, including
39-- a zero gradient at the origin and masking a NaN gradient at negative x.
40constructor PointwisePositiveGate
41
42end-family
43
44family PointwiseExpression : Type 0
45parameter erased pointwiseWord : Type 0
46parameter erased pointwiseInputCount : Nat
47constructor PointwiseInput
48field unrestricted pointwiseInputIndex : (family StdFin pointwiseInputCount)
49constructor PointwiseConstant
50field unrestricted pointwiseConstantWord : pointwiseWord
51constructor PointwiseUnaryExpression
52field unrestricted pointwiseUnaryOperation : PointwiseUnary
53recursive unrestricted pointwiseUnaryOperand
54constructor PointwiseBinaryExpression
55field unrestricted pointwiseBinaryOperation : PointwiseBinary
56recursive unrestricted pointwiseLeftOperand
57recursive unrestricted pointwiseRightOperand
58
59end-family
60
61-- Assignments have sequential meaning: each expression reads the state left
62-- by its predecessor. A learner may explicitly share an intermediate by
63-- assigning a slot; subsequent expressions read that value without rebuilding
64-- its arithmetic tree. This is neither an optimizer catalogue nor a gradient
65-- requirement. Local update rules use exactly the same program structure.
66family PointwiseAssignment : Type 0
67parameter erased pointwiseAssignmentWord : Type 0
68parameter erased pointwiseAssignmentInputs : Nat
69constructor PointwiseAssign
70field unrestricted pointwiseDestination : (family StdFin pointwiseAssignmentInputs)
71field unrestricted pointwiseValue : (family PointwiseExpression pointwiseAssignmentWord pointwiseAssignmentInputs)
72
73end-family
74
75def pointwiseUnaryWith =
76  (lambda unrestricted operations : Float32Operations .
77    unrestricted operation : PointwiseUnary .
78    unrestricted value : Nat .
79    eliminate
80      PointwiseUnary
81      (lambda unrestricted current : PointwiseUnary . Nat)
82      operation
83      (branch PointwiseNegate . f32Negate operations value)
84      (branch PointwiseSquare . f32Multiply operations value value)
85      (branch PointwiseRoundBF16 . bfloat16RoundWord value)
86      (branch PointwiseReciprocal . f32Approximate operations 4 value)
87      (branch PointwiseReciprocalSquareRoot . f32Approximate operations 5 value)
88      (branch PointwiseSquareRoot . f32Approximate operations 6 value)
89      (branch PointwiseTanh . f32Approximate operations 7 value)
90      (branch PointwiseExponentialBase2 . f32Approximate operations 2 value))
91
92def pointwiseBinaryWith =
93  (lambda unrestricted operations : Float32Operations .
94    unrestricted operation : PointwiseBinary .
95    unrestricted left : Nat .
96    unrestricted right : Nat .
97    eliminate
98      PointwiseBinary
99      (lambda unrestricted current : PointwiseBinary . Nat)
100      operation
101      (branch PointwiseAdd . f32Add operations left right)
102      (branch PointwiseMultiply . f32Multiply operations left right)
103      (branch PointwiseSubtract . f32Add operations left (f32Negate operations right))
104      -- closed exactly when x <= 0 (signed zeros included, NaN
105      -- excluded), decided on the word's sign and class: the same
106      -- relation as the ordered comparison with zero, without its
107      -- 2^300-scaled keys, so a host evaluator can run it
108      (branch
109        PointwisePositiveGate
110        .
111        naturalSelect
112          (pipe left
113            modelIsNaN
114            naturalIsZero
115            (naturalAnd (naturalOr (modelSign left) (modelIsZero left))))
116          0
117          right))
118
119-- One structural fold serves semantic evaluators, use analysis and lowerings.
120-- The value domain may differ from the literal word representation: reference
121-- models use exact natural bit patterns, while a lowering uses typed fragments.
122def pointwiseFold =
123  (lambda erased word : Type 0 .
124    erased value : Type 0 .
125    erased inputs : Nat .
126    unrestricted unary : (pi unrestricted operation : PointwiseUnary . unrestricted x : value . value) .
127    unrestricted binary : (pi unrestricted operation : PointwiseBinary . unrestricted x : value . unrestricted y : value . value) .
128    unrestricted literal : (pi unrestricted bits : word . value) .
129    unrestricted expression : (family PointwiseExpression word inputs) .
130    unrestricted values : (pi unrestricted index : (family StdFin inputs) . value) .
131    eliminate
132      PointwiseExpression
133      (lambda unrestricted current : (family PointwiseExpression word inputs) . value)
134      expression
135      (branch PointwiseInput index . values index)
136      (branch PointwiseConstant bits . literal bits)
137      (branch PointwiseUnaryExpression operation operand result . unary operation result)
138      (branch
139        PointwiseBinaryExpression
140        operation
141        left
142        right
143        leftResult
144        rightResult
145        .
146        binary operation leftResult rightResult))
147
148def pointwiseEvaluateWith =
149  (lambda unrestricted operations : Float32Operations .
150    erased inputs : Nat .
151    unrestricted expression : (family PointwiseExpression StdU32 inputs) .
152    unrestricted values : (pi unrestricted i : (family StdFin inputs) . Nat) .
153    pointwiseFold
154      StdU32
155      Nat
156      inputs
157      (pointwiseUnaryWith operations)
158      (pointwiseBinaryWith operations)
159      stdU32ToNatural
160      expression
161      values)
162
163def pointwiseExpressionReads =
164  (lambda erased word : Type 0 .
165    erased inputs : Nat .
166    unrestricted expression : (family PointwiseExpression word inputs) .
167    unrestricted index : Nat .
168    pointwiseFold
169      word
170      Nat
171      inputs
172      (lambda unrestricted operation : PointwiseUnary . unrestricted x : Nat . x)
173      (lambda unrestricted operation : PointwiseBinary . naturalOr)
174      (lambda unrestricted bits : word . 0)
175      expression
176      (lambda unrestricted source : (family StdFin inputs) .
177        naturalEqual index (stdFinToNatural inputs source)))
178
179def pointwiseProgramReadsInitial =
180  (lambda erased word : Type 0 .
181    erased inputs : Nat .
182    unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
183    unrestricted index : Nat .
184    eliminate
185      StdList
186      (lambda unrestricted current : (family StdList (family PointwiseAssignment word inputs)) .
187        Nat)
188      program
189      (branch StdListEmpty . 0)
190      (branch
191        StdListCons
192        head
193        tail
194        rest
195        .
196        eliminate
197          PointwiseAssignment
198          (lambda unrestricted current : (family PointwiseAssignment word inputs) . Nat)
199          head
200          (branch
201            PointwiseAssign
202            destination
203            expression
204            .
205            naturalOr
206              (pointwiseExpressionReads word inputs expression index)
207              (pipe index
208                (naturalEqual (stdFinToNatural inputs destination))
209                naturalIsZero
210                (naturalAnd rest)))))
211
212def pointwiseProgramWrites =
213  (lambda erased word : Type 0 .
214    erased inputs : Nat .
215    unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
216    unrestricted index : Nat .
217    eliminate
218      StdList
219      (lambda unrestricted current : (family StdList (family PointwiseAssignment word inputs)) .
220        Nat)
221      program
222      (branch StdListEmpty . 0)
223      (branch
224        StdListCons
225        head
226        tail
227        rest
228        .
229        eliminate
230          PointwiseAssignment
231          (lambda unrestricted current : (family PointwiseAssignment word inputs) . Nat)
232          head
233          (branch
234            PointwiseAssign
235            destination
236            expression
237            .
238            naturalOr (naturalEqual index (stdFinToNatural inputs destination)) rest)))
239
240def pointwiseProgramEvaluate =
241  (lambda erased word : Type 0 .
242    erased value : Type 0 .
243    erased inputs : Nat .
244    unrestricted unary : (pi unrestricted operation : PointwiseUnary . unrestricted x : value . value) .
245    unrestricted binary : (pi unrestricted operation : PointwiseBinary . unrestricted x : value . unrestricted y : value . value) .
246    unrestricted literal : (pi unrestricted bits : word . value) .
247    unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
248    eliminate
249      StdList
250      (lambda unrestricted current : (family StdList (family PointwiseAssignment word inputs)) .
251        pi unrestricted state : (pi unrestricted index : (family StdFin inputs) . value) .
252          unrestricted index : (family StdFin inputs) .
253          value)
254      program
255      (branch
256        StdListEmpty
257        .
258        lambda unrestricted state : (pi unrestricted index : (family StdFin inputs) . value) .
259          state)
260      (branch
261        StdListCons
262        head
263        tail
264        next
265        .
266        lambda unrestricted state : (pi unrestricted index : (family StdFin inputs) . value) .
267          eliminate
268            PointwiseAssignment
269            (lambda unrestricted current : (family PointwiseAssignment word inputs) .
270              pi unrestricted index : (family StdFin inputs) . value)
271            head
272            (branch
273              PointwiseAssign
274              destination
275              expression
276              .
277              let unrestricted result =
278                (pointwiseFold word value inputs unary binary literal expression state)
279                in
280                next
281                  (lambda unrestricted index : (family StdFin inputs) .
282                    nat-eliminate
283                      (lambda unrestricted same : Nat . value)
284                      (state index)
285                      (lambda unrestricted previous : Nat . unrestricted ignored : value . result)
286                      (naturalEqual
287                        (stdFinToNatural inputs destination)
288                        (stdFinToNatural inputs index))))))
289
290-- Logical slot identities are independent of their eventual physical layout.
291-- Learner compositions use this relation to keep writable state separate from
292-- observations; the target planner separately owns physical non-overlap.
293def pointwiseSlotsSeparate =
294  (lambda erased slots : Nat .
295    unrestricted left : (family StdFin slots) .
296    unrestricted right : (family StdFin slots) .
297    naturalIsZero (naturalEqual (stdFinToNatural slots left) (stdFinToNatural slots right)))
298
299def pointwiseUnary =
300  (lambda erased word : Type 0 .
301    erased inputs : Nat .
302    unrestricted operation : PointwiseUnary .
303    unrestricted value : (family PointwiseExpression word inputs) .
304    constructor PointwiseExpression PointwiseUnaryExpression word inputs operation value)
305
306def pointwiseBinary =
307  (lambda erased word : Type 0 .
308    erased inputs : Nat .
309    unrestricted operation : PointwiseBinary .
310    unrestricted left : (family PointwiseExpression word inputs) .
311    unrestricted right : (family PointwiseExpression word inputs) .
312    constructor PointwiseExpression PointwiseBinaryExpression word inputs operation left right)
313
314-- Each literal or arithmetic node defines one temporary value. Inputs refer
315-- to existing state. This expression traversal is representation-independent;
316-- a target maps values to register pairs or arena words and checks its own
317-- limit. Assignments reuse the largest expression workspace, not their sum.
318def pointwiseTemporaryValues =
319  (lambda erased word : Type 0 .
320    erased inputs : Nat .
321    unrestricted expression : (family PointwiseExpression word inputs) .
322    pointwiseFold
323      word
324      Nat
325      inputs
326      (lambda unrestricted operation : PointwiseUnary . unrestricted child : Nat . succ child)
327      (lambda unrestricted operation : PointwiseBinary .
328        unrestricted left : Nat .
329        unrestricted right : Nat .
330        succ (naturalAdd left right))
331      (lambda unrestricted bits : word . 1)
332      expression
333      (lambda unrestricted index : (family StdFin inputs) . 0))
334
335def pointwiseProgramTemporaryValues =
336  (lambda erased word : Type 0 .
337    erased inputs : Nat .
338    unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
339    stdListFold
340      (family PointwiseAssignment word inputs)
341      Nat
342      (lambda unrestricted assignment : (family PointwiseAssignment word inputs) .
343        unrestricted rest : Nat .
344        eliminate
345          PointwiseAssignment
346          (lambda unrestricted current : (family PointwiseAssignment word inputs) . Nat)
347          assignment
348          (branch
349            PointwiseAssign
350            destination
351            expression
352            .
353            let unrestricted count =
354              (pointwiseTemporaryValues word inputs expression)
355              in
356              naturalSelect (naturalLess count rest) rest count))
357      0
358      program)
359
360-- Squared ReLU and its backward are ordinary compositions, written once
361-- with the compute format's rounding as a parameter (`round` wraps every
362-- value the source materializes in that format). Upstream rounds both
363-- ReLU's result and the square; backward rounds 2*relu(x), then its product
364-- with dy, then applies threshold-backward. Multiplying by a zero mask would
365-- propagate NaNs and is not the same derivative program. In binary32 (no
366-- rounding boundary, nanochat's MLP on the CPU: F.relu(x).square()) the
367-- identity is passed; in BF16 the explicit PointwiseRoundBF16.
368def pointwiseReluSquaredRounded =
369  (lambda erased inputs : Nat .
370    unrestricted round : (pi unrestricted value : (family PointwiseExpression StdU32 inputs) . family PointwiseExpression StdU32 inputs) .
371    unrestricted x : (family PointwiseExpression StdU32 inputs) .
372    let unrestricted relu =
373      (round (pointwiseBinary StdU32 inputs PointwisePositiveGate x x))
374      in
375      round (pointwiseUnary StdU32 inputs PointwiseSquare relu))
376
377def pointwiseReluSquaredBackwardRounded =
378  (lambda erased inputs : Nat .
379    unrestricted round : (pi unrestricted value : (family PointwiseExpression StdU32 inputs) . family PointwiseExpression StdU32 inputs) .
380    unrestricted x : (family PointwiseExpression StdU32 inputs) .
381    unrestricted gradient : (family PointwiseExpression StdU32 inputs) .
382    let unrestricted multiply =
383      (pointwiseBinary StdU32 inputs PointwiseMultiply)
384      in
385      let unrestricted relu =
386        (round (pointwiseBinary StdU32 inputs PointwisePositiveGate x x))
387        in
388        then (pointwiseBinary StdU32 inputs PointwisePositiveGate x)
389          round
390          (multiply gradient)
391          round
392          (multiply (constructor PointwiseExpression PointwiseConstant StdU32 inputs 0x40000000))
393          relu)
394
395def pointwiseRoundBF16Expression =
396  (lambda erased inputs : Nat . pointwiseUnary StdU32 inputs PointwiseRoundBF16)
397
398def pointwiseUnrounded =
399  (lambda erased inputs : Nat . unrestricted value : (family PointwiseExpression StdU32 inputs) .
400    value)
401
402def pointwiseReluSquared =
403  (lambda erased inputs : Nat .
404    pointwiseReluSquaredRounded inputs (pointwiseRoundBF16Expression inputs))
405
406def pointwiseReluSquaredBackward =
407  (lambda erased inputs : Nat .
408    pointwiseReluSquaredBackwardRounded inputs (pointwiseRoundBF16Expression inputs))

The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.