2951def sksPrologue = (lambda unrestricted order : Nat . (lambda unrestricted seq : Nat . (lambda unrestricted heads : Nat . (lambda unrestricted keyValueHeads : Nat . (lambda unrestricted fromConstant : Nat . (lambda unrestricted fromTileBase : Nat .
2952 (lambda unrestricted rest : (family SM86Program) .
2953 (let unrestricted groups = (naturalDivideUnchecked heads keyValueHeads) in
2954 (let unrestricted headBytes = (naturalMultiply seq (naturalMultiply saHeadWidth saHalfBytes)) in
2955 (let unrestricted subTiles = (naturalDivideUnchecked seq skSubTile) in
2956 (saS2R saTid (constructor SM86SpecialRegister SM86ThreadIdX)
2957 (saS2R saTileIndex (saTileAxis order)
2958 (skLoadTileBase fromTileBase
2959 (skLoadHeadIndicesOrdered order fromConstant
2960 (saMovImm saOne 1
2961 (saMovImmAfter saScratch 31 saWait5
2962 (saImad skHead skKeyValueHead groups skHead
2963 (saAnd saLane saTid saScratch
2964 (saShr saWarp saTid 5
2965 (saShr saQuad saLane 2
2966 (saMovImm saScratch 3
2967 (saAnd saQuadLane saLane saScratch
2968 -- the thread's key row: 32 tile + 16 warp + lane / 4
2969 (saImad skRow saWarp 16 saQuad
2970 (saImad skRow saTileIndex skSubTile skRow
2971 -- K's and V's A fragments
2972 (saMovImm saScratch 0
2973 (saImad saScratch skRow (naturalMultiply saHeadWidth saHalfBytes) saScratch
2974 (saImad saScratch saQuadLane 4 saScratch
2975 (saImad saScratch skKeyValueHead headBytes saScratch
2976 (saWide saKeyPointer saScratch saOne (saArgument 1)
2977 (saFor 16 (lambda unrestricted i : Nat .
2978 (let unrestricted kt = (naturalDivideUnchecked i 4) in (let unrestricted j = (naturalModuloUnchecked i 4) in
2979 (saLoad (naturalAdd (skK kt) j) saKeyPointer
2980 (naturalAdd (naturalMultiply kt 32)
2981 (naturalAdd (naturalMultiply (naturalModuloUnchecked j 2) (naturalMultiply 8 (naturalMultiply saHeadWidth saHalfBytes)))
2982 (naturalMultiply (naturalDivideUnchecked j 2) 16)))
2983 saSB1 saWaitNone))))
2984 (saWide saValuePointer saScratch saOne (saArgument 2)
2985 (saFor 16 (lambda unrestricted i : Nat .
2986 (let unrestricted kt = (naturalDivideUnchecked i 4) in (let unrestricted j = (naturalModuloUnchecked i 4) in
2987 (saLoad (naturalAdd (skV kt) j) saValuePointer
2988 (naturalAdd (naturalMultiply kt 32)
2989 (naturalAdd (naturalMultiply (naturalModuloUnchecked j 2) (naturalMultiply 8 (naturalMultiply saHeadWidth saHalfBytes)))
2990 (naturalMultiply (naturalDivideUnchecked j 2) 16)))
2991 saSB1 saWaitNone))))
2992 -- dK's offset (binary32 rows) in saScratch
2993 (saMovImm saScratch 0
2994 (saImad saScratch skRow (naturalMultiply saHeadWidth 4) saScratch
2995 (saImad saScratch saQuadLane 8 saScratch
2996 (saImad saScratch skHead (naturalMultiply seq (naturalMultiply saHeadWidth 4)) saScratch
2997 -- the queries start at the last sub-tile: Q's and dO's rows (row lane / 4,
2998 -- the quad lane's pair), Q^T's and dO^T's columns (row lane / 4), L's and D's
2999 (saMovImm saKeyOffset 0
3000 (saImad saKeyOffset saQuad (naturalMultiply saHeadWidth saHalfBytes) saKeyOffset
3001 (saImad saKeyOffset saQuadLane 4 saKeyOffset
3002 (saImad saKeyOffset skHead headBytes saKeyOffset
3003 (saAddImm saKeyOffset saKeyOffset (naturalMultiply (naturalSaturatingSubtract subTiles 1) (naturalMultiply skSubTile (naturalMultiply saHeadWidth saHalfBytes)))
3004 (saMovImm saValueOffset 0
3005 (saImad saValueOffset saQuad (naturalMultiply seq saHalfBytes) saValueOffset
3006 (saImad saValueOffset saQuadLane 4 saValueOffset
3007 (saImad saValueOffset skHead headBytes saValueOffset
3008 (saAddImm saValueOffset saValueOffset (naturalMultiply (naturalSaturatingSubtract subTiles 1) (naturalMultiply skSubTile saHalfBytes))
3009 (saMovImm skLogOffset 0
3010 (saImad skLogOffset saQuadLane 8 skLogOffset
3011 (saImad skLogOffset skHead (naturalMultiply seq 4) skLogOffset
3012 (saAddImm skLogOffset skLogOffset (naturalMultiply (naturalSaturatingSubtract subTiles 1) (naturalMultiply skSubTile 4))
3013 (sksScoreGradientOffsetAtLast seq
3014 -- the sub-tiles from the last down to the diagonal; the mask's base,
3015 -- 2 (lane % 4) - 16 warp - lane / 4 + 128 - 4096
3016 (saMovImm saCount subTiles
3017 (saImad saCount saTileIndex 4294967295 saCount
3018 (saMovImm saMaskBase 0
3019 (saImad saMaskBase saQuadLane 2 saMaskBase
3020 (saImad saMaskBase saWarp 4294967280 saMaskBase
3021 (saImad saMaskBase saQuad 4294967295 saMaskBase
3022 (saAddImm saMaskBase saMaskBase (naturalSaturatingSubtract 4294967296 (naturalSaturatingSubtract 4096 128))
3023 (saMovConst saScale (saArgument 10)
3024 (saFor 64 (lambda unrestricted i : Nat . (saMovImm (naturalAdd 84 i) 0))
3025 (sksSetup seq headBytes rest)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))The compiler supplied declaration spans and resolved links from this source snapshot. This page does not assert that this file belongs to a checked closure.