Bob.Model

1module Bob.Model
2
3import Component.Linear
4import Representation.Schema
5import Std.Foundation
6import Std.List
7import Std.Natural
8
9-- Bob is a byte bigram, not a small transformer: a raw current byte selects
10-- one row of a learned 256 x 256 logits table for the following byte.
11family BobModelSemantics : Type 0
12constructor BobModelSemanticsValue
13field unrestricted bobModelInputIdentity : Bytes
14field unrestricted bobModelVocabularySize : Nat
15field unrestricted bobModelContextLength : Nat
16field unrestricted bobModelTable : (family LinearComponent)
17field unrestricted bobModelParameters : (family StdList (family ParameterSchema))
18field unrestricted bobModelBiasParameters : Nat
19field unrestricted bobModelPersistentStateValues : Nat
20
21end-family
22
23def bobVocabularySize : Nat =
24  256
25
26def bobContextLength : Nat =
27  1
28
29def bobParameterCount : Nat =
30  (nat-multiply bobVocabularySize bobVocabularySize)
31
32def bobTableComponent : (family LinearComponent) =
33  (constructor
34    LinearComponent
35    LinearComponentOf
36    b"logits-table"
37    bobVocabularySize
38    bobVocabularySize
39    (constructor StdBool StdFalse))
40
41def bobParameterPrefix : (family StdList Bytes) =
42  (constructor StdList StdListCons Bytes b"bob" (constructor StdList StdListEmpty Bytes))
43
44-- Component.Linear derives the one weight schema from the mechanism.  The
45-- false bias field is what makes this list contain exactly one parameter.
46def bobParameters : (family StdList (family ParameterSchema)) =
47  (linearParameters bobParameterPrefix bobTableComponent)
48
49def bobModel : (family BobModelSemantics) =
50  (record
51    BobModelSemantics
52    (bobModelInputIdentity = b"raw-byte-sequence")
53    (bobModelVocabularySize = bobVocabularySize)
54    (bobModelContextLength = bobContextLength)
55    (bobModelTable = bobTableComponent)
56    (bobModelParameters = bobParameters)
57    (bobModelBiasParameters = 0)
58    (bobModelPersistentStateValues = 0))