1module Component.Pointwise
2
3import Data.BFloat16
4import BinaryFloatModel
5import Float32Model
6import FloatLiteralSpec
7import Model.Config
8import Model.Parameter
9import Model.Word32
10import Std.Fin
11import Std.List
12import Std.Natural
13import Std.Word
14
15-- A scalar expression applied independently at each tensor element. The word
16-- type binds literal/input representation; a binary64 backend cannot consume
17-- a binary32 expression by reinterpreting its literals. Input indices are
18-- bounded in the type; hardware, pointers and thread counts do not occur here.
19-- The chosen arithmetic provider owns rounding. An explicit rounding operation
20-- remains a boundary that a backend may not fuse across.
21family PointwiseUnary : Type 0
22constructor PointwiseNegate
23constructor PointwiseSquare
24constructor PointwiseRoundBF16
25constructor PointwiseReciprocal
26constructor PointwiseReciprocalSquareRoot
27constructor PointwiseSquareRoot
28constructor PointwiseTanh
29constructor PointwiseExponentialBase2
30
31end-family
32
33family PointwiseBinary : Type 0
34constructor PointwiseAdd
35constructor PointwiseMultiply
36constructor PointwiseSubtract
37-- threshold(x, other): zero when x <= 0, otherwise other. NaNs take the
38-- other arm. This is the upstream ReLU/threshold-backward rule, including
39-- a zero gradient at the origin and masking a NaN gradient at negative x.
40constructor PointwisePositiveGate
41
42end-family
43
44family PointwiseExpression : Type 0
45parameter erased pointwiseWord : Type 0
46parameter erased pointwiseInputCount : Nat
47constructor PointwiseInput
48field unrestricted pointwiseInputIndex : (family StdFin pointwiseInputCount)
49constructor PointwiseConstant
50field unrestricted pointwiseConstantWord : pointwiseWord
51constructor PointwiseUnaryExpression
52field unrestricted pointwiseUnaryOperation : PointwiseUnary
53recursive unrestricted pointwiseUnaryOperand
54constructor PointwiseBinaryExpression
55field unrestricted pointwiseBinaryOperation : PointwiseBinary
56recursive unrestricted pointwiseLeftOperand
57recursive unrestricted pointwiseRightOperand
58
59end-family
60
61-- Assignments have sequential meaning: each expression reads the state left
62-- by its predecessor. A learner may explicitly share an intermediate by
63-- assigning a slot; subsequent expressions read that value without rebuilding
64-- its arithmetic tree. This is neither an optimizer catalogue nor a gradient
65-- requirement. Local update rules use exactly the same program structure.
66family PointwiseAssignment : Type 0
67parameter erased pointwiseAssignmentWord : Type 0
68parameter erased pointwiseAssignmentInputs : Nat
69constructor PointwiseAssign
70field unrestricted pointwiseDestination : (family StdFin pointwiseAssignmentInputs)
71field unrestricted pointwiseValue : (family PointwiseExpression pointwiseAssignmentWord pointwiseAssignmentInputs)
72
73end-family
74
75def pointwiseUnaryWith =
76 (lambda unrestricted operations : Float32Operations .
77 unrestricted operation : PointwiseUnary .
78 unrestricted value : Nat .
79 eliminate
80 PointwiseUnary
81 (lambda unrestricted current : PointwiseUnary . Nat)
82 operation
83 (branch PointwiseNegate . f32Negate operations value)
84 (branch PointwiseSquare . f32Multiply operations value value)
85 (branch PointwiseRoundBF16 . bfloat16RoundWord value)
86 (branch PointwiseReciprocal . f32Approximate operations 4 value)
87 (branch PointwiseReciprocalSquareRoot . f32Approximate operations 5 value)
88 (branch PointwiseSquareRoot . f32Approximate operations 6 value)
89 (branch PointwiseTanh . f32Approximate operations 7 value)
90 (branch PointwiseExponentialBase2 . f32Approximate operations 2 value))
91
92def pointwiseBinaryWith =
93 (lambda unrestricted operations : Float32Operations .
94 unrestricted operation : PointwiseBinary .
95 unrestricted left : Nat .
96 unrestricted right : Nat .
97 eliminate
98 PointwiseBinary
99 (lambda unrestricted current : PointwiseBinary . Nat)
100 operation
101 (branch PointwiseAdd . f32Add operations left right)
102 (branch PointwiseMultiply . f32Multiply operations left right)
103 (branch PointwiseSubtract . f32Add operations left (f32Negate operations right))
104 -- closed exactly when x <= 0 (signed zeros included, NaN
105 -- excluded), decided on the word's sign and class: the same
106 -- relation as the ordered comparison with zero, without its
107 -- 2^300-scaled keys, so a host evaluator can run it
108 (branch
109 PointwisePositiveGate
110 .
111 naturalSelect
112 (pipe left
113 modelIsNaN
114 naturalIsZero
115 (naturalAnd (naturalOr (modelSign left) (modelIsZero left))))
116 0
117 right))
118
119-- One structural fold serves semantic evaluators, use analysis and lowerings.
120-- The value domain may differ from the literal word representation: reference
121-- models use exact natural bit patterns, while a lowering uses typed fragments.
122def pointwiseFold =
123 (lambda erased word : Type 0 .
124 erased value : Type 0 .
125 erased inputs : Nat .
126 unrestricted unary : (pi unrestricted operation : PointwiseUnary . unrestricted x : value . value) .
127 unrestricted binary : (pi unrestricted operation : PointwiseBinary . unrestricted x : value . unrestricted y : value . value) .
128 unrestricted literal : (pi unrestricted bits : word . value) .
129 unrestricted expression : (family PointwiseExpression word inputs) .
130 unrestricted values : (pi unrestricted index : (family StdFin inputs) . value) .
131 eliminate
132 PointwiseExpression
133 (lambda unrestricted current : (family PointwiseExpression word inputs) . value)
134 expression
135 (branch PointwiseInput index . values index)
136 (branch PointwiseConstant bits . literal bits)
137 (branch PointwiseUnaryExpression operation operand result . unary operation result)
138 (branch
139 PointwiseBinaryExpression
140 operation
141 left
142 right
143 leftResult
144 rightResult
145 .
146 binary operation leftResult rightResult))
147
148def pointwiseEvaluateWith =
149 (lambda unrestricted operations : Float32Operations .
150 erased inputs : Nat .
151 unrestricted expression : (family PointwiseExpression StdU32 inputs) .
152 unrestricted values : (pi unrestricted i : (family StdFin inputs) . Nat) .
153 pointwiseFold
154 StdU32
155 Nat
156 inputs
157 (pointwiseUnaryWith operations)
158 (pointwiseBinaryWith operations)
159 stdU32ToNatural
160 expression
161 values)
162
163def pointwiseExpressionReads =
164 (lambda erased word : Type 0 .
165 erased inputs : Nat .
166 unrestricted expression : (family PointwiseExpression word inputs) .
167 unrestricted index : Nat .
168 pointwiseFold
169 word
170 Nat
171 inputs
172 (lambda unrestricted operation : PointwiseUnary . unrestricted x : Nat . x)
173 (lambda unrestricted operation : PointwiseBinary . naturalOr)
174 (lambda unrestricted bits : word . 0)
175 expression
176 (lambda unrestricted source : (family StdFin inputs) .
177 naturalEqual index (stdFinToNatural inputs source)))
178
179def pointwiseProgramReadsInitial =
180 (lambda erased word : Type 0 .
181 erased inputs : Nat .
182 unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
183 unrestricted index : Nat .
184 eliminate
185 StdList
186 (lambda unrestricted current : (family StdList (family PointwiseAssignment word inputs)) .
187 Nat)
188 program
189 (branch StdListEmpty . 0)
190 (branch
191 StdListCons
192 head
193 tail
194 rest
195 .
196 eliminate
197 PointwiseAssignment
198 (lambda unrestricted current : (family PointwiseAssignment word inputs) . Nat)
199 head
200 (branch
201 PointwiseAssign
202 destination
203 expression
204 .
205 naturalOr
206 (pointwiseExpressionReads word inputs expression index)
207 (pipe index
208 (naturalEqual (stdFinToNatural inputs destination))
209 naturalIsZero
210 (naturalAnd rest)))))
211
212def pointwiseProgramWrites =
213 (lambda erased word : Type 0 .
214 erased inputs : Nat .
215 unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
216 unrestricted index : Nat .
217 eliminate
218 StdList
219 (lambda unrestricted current : (family StdList (family PointwiseAssignment word inputs)) .
220 Nat)
221 program
222 (branch StdListEmpty . 0)
223 (branch
224 StdListCons
225 head
226 tail
227 rest
228 .
229 eliminate
230 PointwiseAssignment
231 (lambda unrestricted current : (family PointwiseAssignment word inputs) . Nat)
232 head
233 (branch
234 PointwiseAssign
235 destination
236 expression
237 .
238 naturalOr (naturalEqual index (stdFinToNatural inputs destination)) rest)))
239
240def pointwiseProgramEvaluate =
241 (lambda erased word : Type 0 .
242 erased value : Type 0 .
243 erased inputs : Nat .
244 unrestricted unary : (pi unrestricted operation : PointwiseUnary . unrestricted x : value . value) .
245 unrestricted binary : (pi unrestricted operation : PointwiseBinary . unrestricted x : value . unrestricted y : value . value) .
246 unrestricted literal : (pi unrestricted bits : word . value) .
247 unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
248 eliminate
249 StdList
250 (lambda unrestricted current : (family StdList (family PointwiseAssignment word inputs)) .
251 pi unrestricted state : (pi unrestricted index : (family StdFin inputs) . value) .
252 unrestricted index : (family StdFin inputs) .
253 value)
254 program
255 (branch
256 StdListEmpty
257 .
258 lambda unrestricted state : (pi unrestricted index : (family StdFin inputs) . value) .
259 state)
260 (branch
261 StdListCons
262 head
263 tail
264 next
265 .
266 lambda unrestricted state : (pi unrestricted index : (family StdFin inputs) . value) .
267 eliminate
268 PointwiseAssignment
269 (lambda unrestricted current : (family PointwiseAssignment word inputs) .
270 pi unrestricted index : (family StdFin inputs) . value)
271 head
272 (branch
273 PointwiseAssign
274 destination
275 expression
276 .
277 let unrestricted result =
278 (pointwiseFold word value inputs unary binary literal expression state)
279 in
280 next
281 (lambda unrestricted index : (family StdFin inputs) .
282 nat-eliminate
283 (lambda unrestricted same : Nat . value)
284 (state index)
285 (lambda unrestricted previous : Nat . unrestricted ignored : value . result)
286 (naturalEqual
287 (stdFinToNatural inputs destination)
288 (stdFinToNatural inputs index))))))
289
290-- Logical slot identities are independent of their eventual physical layout.
291-- Learner compositions use this relation to keep writable state separate from
292-- observations; the target planner separately owns physical non-overlap.
293def pointwiseSlotsSeparate =
294 (lambda erased slots : Nat .
295 unrestricted left : (family StdFin slots) .
296 unrestricted right : (family StdFin slots) .
297 naturalIsZero (naturalEqual (stdFinToNatural slots left) (stdFinToNatural slots right)))
298
299def pointwiseUnary =
300 (lambda erased word : Type 0 .
301 erased inputs : Nat .
302 unrestricted operation : PointwiseUnary .
303 unrestricted value : (family PointwiseExpression word inputs) .
304 constructor PointwiseExpression PointwiseUnaryExpression word inputs operation value)
305
306def pointwiseBinary =
307 (lambda erased word : Type 0 .
308 erased inputs : Nat .
309 unrestricted operation : PointwiseBinary .
310 unrestricted left : (family PointwiseExpression word inputs) .
311 unrestricted right : (family PointwiseExpression word inputs) .
312 constructor PointwiseExpression PointwiseBinaryExpression word inputs operation left right)
313
314-- Each literal or arithmetic node defines one temporary value. Inputs refer
315-- to existing state. This expression traversal is representation-independent;
316-- a target maps values to register pairs or arena words and checks its own
317-- limit. Assignments reuse the largest expression workspace, not their sum.
318def pointwiseTemporaryValues =
319 (lambda erased word : Type 0 .
320 erased inputs : Nat .
321 unrestricted expression : (family PointwiseExpression word inputs) .
322 pointwiseFold
323 word
324 Nat
325 inputs
326 (lambda unrestricted operation : PointwiseUnary . unrestricted child : Nat . succ child)
327 (lambda unrestricted operation : PointwiseBinary .
328 unrestricted left : Nat .
329 unrestricted right : Nat .
330 succ (naturalAdd left right))
331 (lambda unrestricted bits : word . 1)
332 expression
333 (lambda unrestricted index : (family StdFin inputs) . 0))
334
335def pointwiseProgramTemporaryValues =
336 (lambda erased word : Type 0 .
337 erased inputs : Nat .
338 unrestricted program : (family StdList (family PointwiseAssignment word inputs)) .
339 stdListFold
340 (family PointwiseAssignment word inputs)
341 Nat
342 (lambda unrestricted assignment : (family PointwiseAssignment word inputs) .
343 unrestricted rest : Nat .
344 eliminate
345 PointwiseAssignment
346 (lambda unrestricted current : (family PointwiseAssignment word inputs) . Nat)
347 assignment
348 (branch
349 PointwiseAssign
350 destination
351 expression
352 .
353 let unrestricted count =
354 (pointwiseTemporaryValues word inputs expression)
355 in
356 naturalSelect (naturalLess count rest) rest count))
357 0
358 program)
359
360-- Squared ReLU and its backward are ordinary compositions, written once
361-- with the compute format's rounding as a parameter (`round` wraps every
362-- value the source materializes in that format). Upstream rounds both
363-- ReLU's result and the square; backward rounds 2*relu(x), then its product
364-- with dy, then applies threshold-backward. Multiplying by a zero mask would
365-- propagate NaNs and is not the same derivative program. In binary32 (no
366-- rounding boundary, nanochat's MLP on the CPU: F.relu(x).square()) the
367-- identity is passed; in BF16 the explicit PointwiseRoundBF16.
368def pointwiseReluSquaredRounded =
369 (lambda erased inputs : Nat .
370 unrestricted round : (pi unrestricted value : (family PointwiseExpression StdU32 inputs) . family PointwiseExpression StdU32 inputs) .
371 unrestricted x : (family PointwiseExpression StdU32 inputs) .
372 let unrestricted relu =
373 (round (pointwiseBinary StdU32 inputs PointwisePositiveGate x x))
374 in
375 round (pointwiseUnary StdU32 inputs PointwiseSquare relu))
376
377def pointwiseReluSquaredBackwardRounded =
378 (lambda erased inputs : Nat .
379 unrestricted round : (pi unrestricted value : (family PointwiseExpression StdU32 inputs) . family PointwiseExpression StdU32 inputs) .
380 unrestricted x : (family PointwiseExpression StdU32 inputs) .
381 unrestricted gradient : (family PointwiseExpression StdU32 inputs) .
382 let unrestricted multiply =
383 (pointwiseBinary StdU32 inputs PointwiseMultiply)
384 in
385 let unrestricted relu =
386 (round (pointwiseBinary StdU32 inputs PointwisePositiveGate x x))
387 in
388 then (pointwiseBinary StdU32 inputs PointwisePositiveGate x)
389 round
390 (multiply gradient)
391 round
392 (multiply (constructor PointwiseExpression PointwiseConstant StdU32 inputs 0x40000000))
393 relu)
394
395def pointwiseRoundBF16Expression =
396 (lambda erased inputs : Nat . pointwiseUnary StdU32 inputs PointwiseRoundBF16)
397
398def pointwiseUnrounded =
399 (lambda erased inputs : Nat . unrestricted value : (family PointwiseExpression StdU32 inputs) .
400 value)
401
402def pointwiseReluSquared =
403 (lambda erased inputs : Nat .
404 pointwiseReluSquaredRounded inputs (pointwiseRoundBF16Expression inputs))
405
406def pointwiseReluSquaredBackward =
407 (lambda erased inputs : Nat .
408 pointwiseReluSquaredBackwardRounded inputs (pointwiseRoundBF16Expression inputs))The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.