Source/Packages

Model.LearnedPositionDecoder

packages/components/src/Model/LearnedPositionDecoder.alpha

233 lines48 declarations12.8 KiBSHA-256 58c267358560

Complete file · line 104

LearnedPositionDecoder.alpha

Definition view
1module Model.LearnedPositionDecoder
2
3import Std.Natural
4
5-- The GPT-2 decoder family: a tied token embedding plus a learned position
6-- embedding, then pre-LayerNorm blocks (LayerNorm with gain and shift;
7-- multi-head causal attention through one fused, biased Q/K/V projection
8-- and a biased output projection; a biased feed-forward expansion, the
9-- cubic-tanh GELU and a biased reduction), a final LayerNorm, and logits
10-- through the tied embedding. Radford et al. 2019 and the released
11-- openai-community/gpt2 config.json fix the instance (Model.GPT2).
12--
13-- `vocabulary` is the stored class count, `tokens` the classes that exist:
14-- a realization may pad the vocabulary to a multiple its objective needs;
15-- padded classes never occur as inputs or targets and a realization must
16-- hold their probabilities at zero.
17family LearnedPositionDecoderGeometry : Type 0
18constructor LearnedPositionDecoderGeometryValue
19field unrestricted learnedPositionDecoderVocabulary : Nat
20field unrestricted learnedPositionDecoderTokens : Nat
21field unrestricted learnedPositionDecoderContext : Nat
22field unrestricted learnedPositionDecoderLayers : Nat
23field unrestricted learnedPositionDecoderWidth : Nat
24field unrestricted learnedPositionDecoderHeads : Nat
25field unrestricted learnedPositionDecoderHeadWidth : Nat
26field unrestricted learnedPositionDecoderFeedForwardWidth : Nat
27
28end-family
29
30def learnedPositionDecoderVocabularyOf =
31  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
32    eliminate LearnedPositionDecoderGeometry
33      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
34      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . vocab))
35def learnedPositionDecoderTokensOf =
36  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
37    eliminate LearnedPositionDecoderGeometry
38      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
39      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . tokens))
40def learnedPositionDecoderContextOf =
41  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
42    eliminate LearnedPositionDecoderGeometry
43      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
44      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . context))
45def learnedPositionDecoderLayersOf =
46  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
47    eliminate LearnedPositionDecoderGeometry
48      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
49      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . layers))
50def learnedPositionDecoderWidthOf =
51  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
52    eliminate LearnedPositionDecoderGeometry
53      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
54      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . width))
55def learnedPositionDecoderHeadsOf =
56  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
57    eliminate LearnedPositionDecoderGeometry
58      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
59      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . heads))
60def learnedPositionDecoderHeadWidthOf =
61  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
62    eliminate LearnedPositionDecoderGeometry
63      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
64      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . head))
65def learnedPositionDecoderFeedForwardWidthOf =
66  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
67    eliminate LearnedPositionDecoderGeometry
68      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
69      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed . feed))
70
71-- the fused Q/K/V projection's output width
72def learnedPositionDecoderProjectionWidth =
73  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
74    naturalMultiply 3 (learnedPositionDecoderWidthOf geometry))
75
76-- the heads tile the width; padding never removes a real class
77def learnedPositionDecoderAdmitted =
78  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
79    eliminate LearnedPositionDecoderGeometry
80      (lambda unrestricted current : LearnedPositionDecoderGeometry . Nat) geometry
81      (branch LearnedPositionDecoderGeometryValue vocab tokens context layers width heads head feed .
82        nest naturalAnd (naturalNonzero tokens)
83          (naturalLessOrEqual tokens vocab)
84            (naturalNonzero context)
85              (naturalNonzero layers)
86                (naturalNonzero heads)
87                  (naturalEqual width (naturalMultiply heads head))
88                    (naturalNonzero feed)))
89
90-- ---- the parameter layout ----
91-- Element offsets, independent of F32/BF16 storage, in two contiguous
92-- regions so a learner can treat them differently (GPT-2 practice decays
93-- the matrices and embeddings, not the biases and LayerNorm parameters):
94--   matrices: the token embedding [vocabulary][width], the position
95--     embedding [context][width], then per block the Q/K/V weight
96--     [3 width][width], the attention output weight [width][width], the
97--     expansion weight [feed][width] and the reduction weight [width][feed];
98--   vectors: per block the attention LayerNorm's gain and shift, the Q/K/V
99--     bias, the output bias, the feed LayerNorm's gain and shift, the
100--     expansion bias and the reduction bias; then the final LayerNorm's gain
101--     and shift.
102-- A weight is stored [output][input], as the forward product reads it.
103def learnedPositionDecoderEmbeddingBase : Nat = 0
104def learnedPositionDecoderEmbeddingElements =
105  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
106    naturalMultiply (learnedPositionDecoderVocabularyOf geometry)
107      (learnedPositionDecoderWidthOf geometry))
108def learnedPositionDecoderPositionBase = learnedPositionDecoderEmbeddingElements
109def learnedPositionDecoderPositionElements =
110  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
111    naturalMultiply (learnedPositionDecoderContextOf geometry)
112      (learnedPositionDecoderWidthOf geometry))
113def learnedPositionDecoderProjectionElements =
114  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
115    naturalMultiply (learnedPositionDecoderProjectionWidth geometry)
116      (learnedPositionDecoderWidthOf geometry))
117def learnedPositionDecoderOutputElements =
118  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
119    naturalMultiply (learnedPositionDecoderWidthOf geometry)
120      (learnedPositionDecoderWidthOf geometry))
121def learnedPositionDecoderFeedElements =
122  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
123    naturalMultiply (learnedPositionDecoderFeedForwardWidthOf geometry)
124      (learnedPositionDecoderWidthOf geometry))
125
126-- ---- the matrices ----
127def learnedPositionDecoderBlockMatrixElements =
128  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
129    nest naturalAdd (learnedPositionDecoderProjectionElements geometry)
130      (learnedPositionDecoderOutputElements geometry)
131        (naturalMultiply 2 (learnedPositionDecoderFeedElements geometry)))
132def learnedPositionDecoderProjectionBase =
133  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
134    unrestricted layer : Nat .
135      nest naturalAdd (learnedPositionDecoderPositionBase geometry)
136        (learnedPositionDecoderPositionElements geometry)
137          (naturalMultiply layer (learnedPositionDecoderBlockMatrixElements geometry)))
138def learnedPositionDecoderOutputBase =
139  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
140    unrestricted layer : Nat .
141      naturalAdd (learnedPositionDecoderProjectionBase geometry layer)
142        (learnedPositionDecoderProjectionElements geometry))
143def learnedPositionDecoderFeedUpBase =
144  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
145    unrestricted layer : Nat .
146      naturalAdd (learnedPositionDecoderOutputBase geometry layer)
147        (learnedPositionDecoderOutputElements geometry))
148def learnedPositionDecoderFeedDownBase =
149  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
150    unrestricted layer : Nat .
151      naturalAdd (learnedPositionDecoderFeedUpBase geometry layer)
152        (learnedPositionDecoderFeedElements geometry))
153-- the first vector, and the matrices' count
154def learnedPositionDecoderVectorBase =
155  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
156    learnedPositionDecoderProjectionBase geometry (learnedPositionDecoderLayersOf geometry))
157def learnedPositionDecoderMatrixElements = learnedPositionDecoderVectorBase
158
159-- ---- the vectors ----
160def learnedPositionDecoderBlockVectorElements =
161  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
162    nest naturalAdd (naturalMultiply 6 (learnedPositionDecoderWidthOf geometry))
163      (learnedPositionDecoderProjectionWidth geometry)
164        (learnedPositionDecoderFeedForwardWidthOf geometry))
165def learnedPositionDecoderNormInputGainBase =
166  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
167    unrestricted layer : Nat .
168      naturalAdd (learnedPositionDecoderVectorBase geometry)
169        (naturalMultiply layer (learnedPositionDecoderBlockVectorElements geometry)))
170def learnedPositionDecoderNormInputShiftBase =
171  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
172    unrestricted layer : Nat .
173      naturalAdd (learnedPositionDecoderNormInputGainBase geometry layer)
174        (learnedPositionDecoderWidthOf geometry))
175def learnedPositionDecoderProjectionBiasBase =
176  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
177    unrestricted layer : Nat .
178      naturalAdd (learnedPositionDecoderNormInputShiftBase geometry layer)
179        (learnedPositionDecoderWidthOf geometry))
180def learnedPositionDecoderOutputBiasBase =
181  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
182    unrestricted layer : Nat .
183      naturalAdd (learnedPositionDecoderProjectionBiasBase geometry layer)
184        (learnedPositionDecoderProjectionWidth geometry))
185def learnedPositionDecoderNormFeedGainBase =
186  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
187    unrestricted layer : Nat .
188      naturalAdd (learnedPositionDecoderOutputBiasBase geometry layer)
189        (learnedPositionDecoderWidthOf geometry))
190def learnedPositionDecoderNormFeedShiftBase =
191  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
192    unrestricted layer : Nat .
193      naturalAdd (learnedPositionDecoderNormFeedGainBase geometry layer)
194        (learnedPositionDecoderWidthOf geometry))
195def learnedPositionDecoderFeedUpBiasBase =
196  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
197    unrestricted layer : Nat .
198      naturalAdd (learnedPositionDecoderNormFeedShiftBase geometry layer)
199        (learnedPositionDecoderWidthOf geometry))
200def learnedPositionDecoderFeedDownBiasBase =
201  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
202    unrestricted layer : Nat .
203      naturalAdd (learnedPositionDecoderFeedUpBiasBase geometry layer)
204        (learnedPositionDecoderFeedForwardWidthOf geometry))
205def learnedPositionDecoderFinalNormGainBase =
206  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
207    learnedPositionDecoderNormInputGainBase geometry (learnedPositionDecoderLayersOf geometry))
208def learnedPositionDecoderFinalNormShiftBase =
209  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
210    naturalAdd (learnedPositionDecoderFinalNormGainBase geometry)
211      (learnedPositionDecoderWidthOf geometry))
212def learnedPositionDecoderParameterCount =
213  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
214    naturalAdd (learnedPositionDecoderFinalNormShiftBase geometry)
215      (learnedPositionDecoderWidthOf geometry))
216def learnedPositionDecoderVectorElements =
217  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
218    naturalSaturatingSubtract (learnedPositionDecoderParameterCount geometry)
219      (learnedPositionDecoderVectorBase geometry))
220
221-- both regions are dense: a block's last matrix and vector end where the
222-- next block's begin
223def learnedPositionDecoderLayoutDense =
224  (lambda unrestricted geometry : LearnedPositionDecoderGeometry .
225    naturalAnd
226      (pipe geometry
227        (learnedPositionDecoderFeedDownBase 0)
228        (naturalAdd (learnedPositionDecoderFeedElements geometry))
229        (naturalEqual (learnedPositionDecoderProjectionBase geometry 1)))
230      (pipe geometry
231        (learnedPositionDecoderFeedDownBiasBase 0)
232        (naturalAdd (learnedPositionDecoderWidthOf geometry))
233        (naturalEqual (learnedPositionDecoderNormInputGainBase geometry 1))))

The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.