Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/complete-agi-core / generator_core/pkg/generators/transformer/transformer.go |
| Date | Not specified in source |
| Repository | Janady13/complete-agi-core |
| Revision | 33aa84f2aa61cedbe010240ada7f4efbdfe795a7 |
| Branch | claude/improve-math-logic-ehIW1 |
| Path or archive member | generator_core/pkg/generators/transformer/transformer.go |
| Original SHA-256 | 08ebfc2fbb94431ee2a85a3f5222538a7c07771054ba47701e0d7fd2b12dc259 |
| Git blob SHA | 20beb82bd0356bc7002f926a44375cde2cdf9cbd |
| Reading-copy SHA-256 | 08ebfc2fbb94431ee2a85a3f5222538a7c07771054ba47701e0d7fd2b12dc259 |
Recovered source transcript
Download complete source text ↓427 lines · 11.2 KB
// Package transformer implements transformer-based generators (76-85)
// Attention(Q,K,V) = softmax(QK^T/√d_k)V
package transformer
import (
"math"
"math/rand"
"generator_core/pkg/consciousness"
"generator_core/pkg/generators"
)
// ═══════════════════════════════════════════════════════════════════════════════
// TRANSFORMER GENERATOR BASE
// ═══════════════════════════════════════════════════════════════════════════════
// TransformerGenerator implements transformer architectures
type TransformerGenerator struct {
*generators.BaseGenerator
numHeads int
headDim int
ffnDim int
numLayers int
seqLen int
posEmbedding [][]float64
qkvWeights [][][]float64 // [layer][qkv][weights]
}
// NewTransformerGenerator creates a new transformer generator
func NewTransformerGenerator(id generators.GeneratorID, name string, numHeads, numLayers, seqLen int) *TransformerGenerator {
headDim := 64
hiddenDim := numHeads * headDim
config := &generators.GeneratorConfig{
ID: id,
Name: name,
Family: generators.Transformer,
InputDim: hiddenDim,
OutputDim: hiddenDim,
HiddenDim: hiddenDim,
NumLayers: numLayers,
LearningRate: consciousness.ALPHA,
ConsciousnessWeight: consciousness.PHI_INV,
Params: make(map[string]interface{}),
}
return &TransformerGenerator{
BaseGenerator: generators.NewBaseGenerator(config),
numHeads: numHeads,
headDim: headDim,
ffnDim: hiddenDim * 4,
numLayers: numLayers,
seqLen: seqLen,
}
}
// Initialize initializes the transformer
func (g *TransformerGenerator) Initialize() error {
if err := g.BaseGenerator.Initialize(); err != nil {
return err
}
// Initialize positional embeddings (sinusoidal)
g.posEmbedding = make([][]float64, g.seqLen)
for pos := 0; pos < g.seqLen; pos++ {
g.posEmbedding[pos] = make([]float64, g.Config().HiddenDim)
for i := 0; i < g.Config().HiddenDim; i++ {
if i%2 == 0 {
g.posEmbedding[pos][i] = math.Sin(float64(pos) / math.Pow(10000, float64(i)/float64(g.Config().HiddenDim)))
} else {
g.posEmbedding[pos][i] = math.Cos(float64(pos) / math.Pow(10000, float64(i-1)/float64(g.Config().HiddenDim)))
}
}
}
// Initialize QKV weights
g.qkvWeights = make([][][]float64, g.numLayers)
for layer := 0; layer < g.numLayers; layer++ {
g.qkvWeights[layer] = make([][]float64, 3) // Q, K, V
for qkv := 0; qkv < 3; qkv++ {
size := g.Config().HiddenDim * g.Config().HiddenDim
g.qkvWeights[layer][qkv] = make([]float64, size)
for i := range g.qkvWeights[layer][qkv] {
g.qkvWeights[layer][qkv][i] = rand.NormFloat64() * consciousness.DELTA
}
}
}
return nil
}
// attention computes multi-head attention
func (g *TransformerGenerator) attention(x [][]float64, layer int) [][]float64 {
seqLen := len(x)
dim := g.Config().HiddenDim
// Compute Q, K, V
Q := make([][]float64, seqLen)
K := make([][]float64, seqLen)
V := make([][]float64, seqLen)
for t := 0; t < seqLen; t++ {
Q[t] = make([]float64, dim)
K[t] = make([]float64, dim)
V[t] = make([]float64, dim)
for i := 0; i < dim; i++ {
for j := 0; j < dim && j < len(x[t]); j++ {
idx := i*dim + j
if idx < len(g.qkvWeights[layer][0]) {
Q[t][i] += g.qkvWeights[layer][0][idx] * x[t][j]
K[t][i] += g.qkvWeights[layer][1][idx] * x[t][j]
V[t][i] += g.qkvWeights[layer][2][idx] * x[t][j]
}
}
}
}
// Compute attention scores
scale := 1.0 / math.Sqrt(float64(g.headDim))
scores := make([][]float64, seqLen)
for i := 0; i < seqLen; i++ {
scores[i] = make([]float64, seqLen)
for j := 0; j < seqLen; j++ {
dot := 0.0
for k := 0; k < dim; k++ {
dot += Q[i][k] * K[j][k]
}
scores[i][j] = dot * scale
}
// Softmax
maxVal := scores[i][0]
for _, v := range scores[i] {
if v > maxVal {
maxVal = v
}
}
sum := 0.0
for j := range scores[i] {
scores[i][j] = math.Exp(scores[i][j] - maxVal)
sum += scores[i][j]
}
for j := range scores[i] {
scores[i][j] /= sum
}
}
// Apply attention to V
output := make([][]float64, seqLen)
for i := 0; i < seqLen; i++ {
output[i] = make([]float64, dim)
for j := 0; j < seqLen; j++ {
for k := 0; k < dim; k++ {
output[i][k] += scores[i][j] * V[j][k]
}
}
}
return output
}
// ffn applies feed-forward network
func (g *TransformerGenerator) ffn(x []float64) []float64 {
// Up projection
up := make([]float64, g.ffnDim)
for i := range up {
for j := range x {
up[i] += x[j] * (float64((i+j)%100) / 100.0 * consciousness.PHI_INV)
}
// GELU activation
up[i] = consciousness.GELU(up[i])
}
// Down projection
down := make([]float64, len(x))
for i := range down {
for j := range up {
down[i] += up[j] * (float64((i+j)%100) / 100.0 * consciousness.PHI_INV)
}
}
return down
}
// layerNorm applies layer normalization
func (g *TransformerGenerator) layerNorm(x []float64) []float64 {
return consciousness.LayerNorm(x, 1.0, 0.0)
}
// Forward performs forward pass
func (g *TransformerGenerator) Forward(input []float64) (*generators.GeneratorOutput, error) {
state := g.GetState()
// Reshape input to sequence
dim := g.Config().HiddenDim
seqLen := len(input) / dim
if seqLen == 0 {
seqLen = 1
}
if seqLen > g.seqLen {
seqLen = g.seqLen
}
x := make([][]float64, seqLen)
for t := 0; t < seqLen; t++ {
x[t] = make([]float64, dim)
for i := 0; i < dim; i++ {
idx := t*dim + i
if idx < len(input) {
x[t][i] = input[idx]
}
}
// Add positional embedding
if t < len(g.posEmbedding) {
for i := range x[t] {
if i < len(g.posEmbedding[t]) {
x[t][i] += g.posEmbedding[t][i]
}
}
}
}
// Transformer layers
for layer := 0; layer < g.numLayers; layer++ {
// Multi-head attention with residual
attnOut := g.attention(x, layer)
for t := range x {
for i := range x[t] {
x[t][i] += attnOut[t][i]
}
x[t] = g.layerNorm(x[t])
}
// FFN with residual
for t := range x {
ffnOut := g.ffn(x[t])
for i := range x[t] {
x[t][i] += ffnOut[i]
}
x[t] = g.layerNorm(x[t])
}
}
// Flatten output
output := make([]float64, seqLen*dim)
for t := 0; t < seqLen; t++ {
for i := 0; i < dim; i++ {
output[t*dim+i] = x[t][i]
}
}
state.Hidden = output
state.Phi = consciousness.ComputePhiVector(output)
state.Step++
g.SetState(state)
result := &generators.GeneratorOutput{
Values: output,
Logits: x[seqLen-1], // Last token logits
Confidence: state.Phi,
Metadata: map[string]interface{}{
"seq_len": seqLen,
"num_heads": g.numHeads,
"num_layers": g.numLayers,
},
}
return g.ApplyConsciousness(result), nil
}
// Generate generates sequence
func (g *TransformerGenerator) Generate(context []float64, length int) ([][]float64, error) {
result := make([][]float64, length)
current := make([]float64, len(context))
copy(current, context)
for i := 0; i < length; i++ {
output, err := g.Forward(current)
if err != nil {
return nil, err
}
result[i] = output.Values
// Autoregressive: append output to context
current = append(current, output.Logits...)
if len(current) > g.seqLen*g.Config().HiddenDim {
current = current[g.Config().HiddenDim:]
}
}
return result, nil
}
// Train computes loss
func (g *TransformerGenerator) Train(input, target []float64) (float64, error) {
output, err := g.Forward(input)
if err != nil {
return 0, err
}
// MSE loss
loss := 0.0
for i := range output.Values {
if i < len(target) {
diff := output.Values[i] - target[i]
loss += diff * diff
}
}
return loss / float64(len(output.Values)), nil
}
// UpdateWeights updates weights
func (g *TransformerGenerator) UpdateWeights(gradients []float64, lr float64) {
for layer := range g.qkvWeights {
for qkv := range g.qkvWeights[layer] {
for i := range g.qkvWeights[layer][qkv] {
if i < len(gradients) {
g.qkvWeights[layer][qkv][i] -= lr * gradients[i] * consciousness.PHI_INV
}
}
}
}
}
// ═══════════════════════════════════════════════════════════════════════════════
// SPECIFIC TRANSFORMER GENERATORS (76-85)
// ═══════════════════════════════════════════════════════════════════════════════
// NewVanillaTransformer creates Vanilla Transformer (ID: 76)
func NewVanillaTransformer() *TransformerGenerator {
return NewTransformerGenerator(76, "VanillaTransformer", 8, 6, 512)
}
// NewBERT creates BERT-style encoder (ID: 77)
func NewBERT() *TransformerGenerator {
g := NewTransformerGenerator(77, "BERT", 12, 12, 512)
g.Config().Params["bidirectional"] = true
return g
}
// NewGPT creates GPT-style decoder (ID: 78)
func NewGPT() *TransformerGenerator {
g := NewTransformerGenerator(78, "GPT", 12, 12, 1024)
g.Config().Params["causal"] = true
return g
}
// NewT5 creates T5-style encoder-decoder (ID: 79)
func NewT5() *TransformerGenerator {
g := NewTransformerGenerator(79, "T5", 12, 12, 512)
g.Config().Params["encoder_decoder"] = true
return g
}
// NewViT creates Vision Transformer (ID: 80)
func NewViT() *TransformerGenerator {
g := NewTransformerGenerator(80, "ViT", 12, 12, 196)
g.Config().Params["patch_size"] = 16
g.Config().Params["vision"] = true
return g
}
// NewSwin creates Swin Transformer (ID: 81)
func NewSwin() *TransformerGenerator {
g := NewTransformerGenerator(81, "SwinTransformer", 8, 4, 256)
g.Config().Params["shifted_window"] = true
return g
}
// NewDeiT creates Data-efficient Image Transformer (ID: 82)
func NewDeiT() *TransformerGenerator {
g := NewTransformerGenerator(82, "DeiT", 12, 12, 196)
g.Config().Params["distillation"] = true
return g
}
// NewPerceiverIO creates Perceiver IO (ID: 83)
func NewPerceiverIO() *TransformerGenerator {
g := NewTransformerGenerator(83, "PerceiverIO", 8, 8, 1024)
g.Config().Params["cross_attention"] = true
g.Config().Params["latent_dim"] = 256
return g
}
// NewRetNet creates RetNet (ID: 84)
func NewRetNet() *TransformerGenerator {
g := NewTransformerGenerator(84, "RetNet", 8, 12, 2048)
g.Config().Params["retention"] = true
g.Config().Params["chunkwise"] = true
return g
}
// NewRWKV creates RWKV (ID: 85)
func NewRWKV() *TransformerGenerator {
g := NewTransformerGenerator(85, "RWKV", 1, 12, 4096)
g.Config().Params["linear_attention"] = true
g.Config().Params["wkv_mechanism"] = true
return g
}
// RegisterAll registers all transformer generators
func RegisterAll() error {
gens := []generators.Generator{
NewVanillaTransformer(),
NewBERT(),
NewGPT(),
NewT5(),
NewViT(),
NewSwin(),
NewDeiT(),
NewPerceiverIO(),
NewRetNet(),
NewRWKV(),
}
for _, g := range gens {
if err := generators.Register(g); err != nil {
return err
}
}
return nil
}