Coppelius.Model

1module Coppelius.Model
2
3import Coppelius.Learner
4import Model.Architecture
5import Model.Config
6import Model.Parameter
7import Model.Word32
8import Std.Word
9
10-- Coppelius is the 60M-class dense attention transformer inspired by
11-- huggingface.co/ajaxdavis/alpha-60m-base: 16 layers, width 512, 8 heads of
12-- 64, FFN 1408, vocabulary 12288, context 1024, RoPE, RMSNorm, tied
13-- embeddings, no matrix biases, and an exact-vocabulary objective.
14--
15-- The physically qualified SM86 realization recorded in docs/WORKLOG.md uses
16-- LayerNorm and a gated GELU feed-forward block because those are the
17-- maintained SM86 owners. The reference family remains explicit here rather
18-- than silently encoding realization-specific padding or substitutions.
19def modelCoppeliusVocab : StdU32 =
20  12_288
21
22def modelCoppeliusBlock : StdU32 =
23  1_024
24
25def modelCoppeliusLayers : StdU32 =
26  16
27
28def modelCoppeliusWidth : StdU32 =
29  512
30
31def modelCoppeliusHeads : StdU32 =
32  8
33
34def modelCoppeliusFfnWidth : StdU32 =
35  1_408
36
37def modelCoppeliusOne : StdU32 =
38  1
39
40-- the rotary positions' base: head-dimension pair i of a head width w turns
41-- at theta_i = base^(-2i / w) radians per position
42def coppeliusRotaryBase : Nat =
43  10000
44
45-- a fresh model's matrices (the embedding, and every layer's projections)
46-- are drawn with standard deviation 1 / this; the norms' gains are one and
47-- their shifts zero
48def coppeliusInitializationDeviationDenominator : Nat =
49  50
50
51def modelCoppelius =
52  (record
53    ModelConfig
54    (modelVocabSize = modelCoppeliusVocab)
55    (modelBlockSize = modelCoppeliusBlock)
56    (modelLayers = modelCoppeliusLayers)
57    (modelWidth = modelCoppeliusWidth)
58    (modelHeads = modelCoppeliusHeads)
59    (modelFfnStoredWidth = modelCoppeliusFfnWidth)
60    (modelExperts = modelCoppeliusOne)
61    (modelTopK = modelCoppeliusOne)
62    (modelRoute = (constructor ModelRouteKind ModelCyclicRoute))
63    (modelStackedExperts = (constructor ModelBoolean ModelFalse))
64    (modelHeadRank = (constructor ModelOptionalWord32 ModelNoWord32))
65    (modelAttentionRank = (constructor ModelOptionalWord32 ModelNoWord32))
66    (modelPositionEncoding = (constructor ModelPositionEncoding ModelRotaryPositions))
67    (modelNormKind = (constructor ModelNormKind ModelRMSNormalization))
68    (modelTieEmbeddings = (constructor ModelBoolean ModelTrue))
69    (modelObjective = coppeliusObjective))