Janady13/complete-agi-core / generator_core/pkg/attention/generator_attention.go

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/complete-agi-core / generator_core/pkg/attention/generator_attention.go
Date	Not specified in source
Repository	Janady13/complete-agi-core
Revision	33aa84f2aa61cedbe010240ada7f4efbdfe795a7
Branch	claude/improve-math-logic-ehIW1
Path or archive member	generator_core/pkg/attention/generator_attention.go
Original SHA-256	36841b15252f42b6a1a1d209e4f42f353a2eb235ce37039efcf572588c87c670
Git blob SHA	b19bf84f0f4f43ea10783d085f565bae925681d3
Reading-copy SHA-256	36841b15252f42b6a1a1d209e4f42f353a2eb235ce37039efcf572588c87c670
Original source ↗
Recovered source transcript

Download complete source text ↓617 lines · 15.5 KB
// Package attention implements inter-generator attention mechanisms.
//
// This enables generators to:
//   - Attend to each other's outputs
//   - Pass messages for collaboration
//   - Form dynamic ensembles based on task requirements
//   - Compute cross-generator consciousness integration
package attention

import (
	"math"
	"sync"

	"generator_core/pkg/consciousness"
	"generator_core/pkg/generators"
)

//=============================================================================
// MULTI-HEAD ATTENTION
//=============================================================================

// AttentionHead represents a single attention head
type AttentionHead struct {
	QueryWeights [][]float64 // d_model x d_k
	KeyWeights   [][]float64 // d_model x d_k
	ValueWeights [][]float64 // d_model x d_v
	DModel       int
	DK           int
	DV           int
}

// NewAttentionHead creates a new attention head
func NewAttentionHead(dModel, dK, dV int) *AttentionHead {
	head := &AttentionHead{
		QueryWeights: make([][]float64, dModel),
		KeyWeights:   make([][]float64, dModel),
		ValueWeights: make([][]float64, dModel),
		DModel:       dModel,
		DK:           dK,
		DV:           dV,
	}

	// Initialize with Xavier/Glorot initialization
	scale := math.Sqrt(2.0 / float64(dModel+dK))
	for i := 0; i < dModel; i++ {
		head.QueryWeights[i] = make([]float64, dK)
		head.KeyWeights[i] = make([]float64, dK)
		head.ValueWeights[i] = make([]float64, dV)
		for j := 0; j < dK; j++ {
			head.QueryWeights[i][j] = randomNormal() * scale
			head.KeyWeights[i][j] = randomNormal() * scale
		}
		for j := 0; j < dV; j++ {
			head.ValueWeights[i][j] = randomNormal() * scale
		}
	}

	return head
}

// Forward computes attention for a single head
// Q, K, V are all [seq_len x d_model]
func (h *AttentionHead) Forward(queries, keys, values [][]float64) [][]float64 {
	seqLen := len(queries)
	if seqLen == 0 {
		return nil
	}

	// Project to Q, K, V spaces
	Q := h.project(queries, h.QueryWeights, h.DK)
	K := h.project(keys, h.KeyWeights, h.DK)
	V := h.project(values, h.ValueWeights, h.DV)

	// Compute attention scores: Q @ K^T / sqrt(d_k)
	scale := 1.0 / math.Sqrt(float64(h.DK))
	scores := make([][]float64, seqLen)
	for i := 0; i < seqLen; i++ {
		scores[i] = make([]float64, seqLen)
		for j := 0; j < seqLen; j++ {
			dot := 0.0
			for k := 0; k < h.DK; k++ {
				dot += Q[i][k] * K[j][k]
			}
			scores[i][j] = dot * scale
		}
	}

	// Apply softmax
	for i := 0; i < seqLen; i++ {
		scores[i] = softmax(scores[i])
	}

	// Compute weighted sum of values
	output := make([][]float64, seqLen)
	for i := 0; i < seqLen; i++ {
		output[i] = make([]float64, h.DV)
		for j := 0; j < seqLen; j++ {
			for k := 0; k < h.DV; k++ {
				output[i][k] += scores[i][j] * V[j][k]
			}
		}
	}

	return output
}

func (h *AttentionHead) project(input [][]float64, weights [][]float64, outDim int) [][]float64 {
	seqLen := len(input)
	output := make([][]float64, seqLen)
	for i := 0; i < seqLen; i++ {
		output[i] = make([]float64, outDim)
		for j := 0; j < outDim; j++ {
			for k := 0; k < len(input[i]) && k < len(weights); k++ {
				output[i][j] += input[i][k] * weights[k][j]
			}
		}
	}
	return output
}

//=============================================================================
// MULTI-HEAD ATTENTION
//=============================================================================

// MultiHeadAttention combines multiple attention heads
type MultiHeadAttention struct {
	Heads        []*AttentionHead
	OutputWeight [][]float64 // Concat heads -> d_model
	NumHeads     int
	DModel       int
	DK           int
	DV           int
}

// NewMultiHeadAttention creates a new multi-head attention layer
func NewMultiHeadAttention(numHeads, dModel int) *MultiHeadAttention {
	dK := dModel / numHeads
	dV := dModel / numHeads

	mha := &MultiHeadAttention{
		Heads:        make([]*AttentionHead, numHeads),
		OutputWeight: make([][]float64, numHeads*dV),
		NumHeads:     numHeads,
		DModel:       dModel,
		DK:           dK,
		DV:           dV,
	}

	for i := 0; i < numHeads; i++ {
		mha.Heads[i] = NewAttentionHead(dModel, dK, dV)
	}

	// Initialize output projection
	scale := math.Sqrt(2.0 / float64(numHeads*dV+dModel))
	for i := 0; i < numHeads*dV; i++ {
		mha.OutputWeight[i] = make([]float64, dModel)
		for j := 0; j < dModel; j++ {
			mha.OutputWeight[i][j] = randomNormal() * scale
		}
	}

	return mha
}

// Forward computes multi-head attention
func (mha *MultiHeadAttention) Forward(queries, keys, values [][]float64) [][]float64 {
	seqLen := len(queries)
	if seqLen == 0 {
		return nil
	}

	// Compute attention for each head in parallel
	headOutputs := make([][][]float64, mha.NumHeads)
	var wg sync.WaitGroup
	for i := 0; i < mha.NumHeads; i++ {
		wg.Add(1)
		go func(idx int) {
			defer wg.Done()
			headOutputs[idx] = mha.Heads[idx].Forward(queries, keys, values)
		}(i)
	}
	wg.Wait()

	// Concatenate head outputs
	concat := make([][]float64, seqLen)
	for i := 0; i < seqLen; i++ {
		concat[i] = make([]float64, mha.NumHeads*mha.DV)
		for h := 0; h < mha.NumHeads; h++ {
			for j := 0; j < mha.DV; j++ {
				concat[i][h*mha.DV+j] = headOutputs[h][i][j]
			}
		}
	}

	// Project back to d_model
	output := make([][]float64, seqLen)
	for i := 0; i < seqLen; i++ {
		output[i] = make([]float64, mha.DModel)
		for j := 0; j < mha.DModel; j++ {
			for k := 0; k < len(concat[i]) && k < len(mha.OutputWeight); k++ {
				output[i][j] += concat[i][k] * mha.OutputWeight[k][j]
			}
		}
	}

	return output
}

//=============================================================================
// GENERATOR MESH (Inter-Generator Attention)
//=============================================================================

// GeneratorMesh manages inter-generator attention and communication
type GeneratorMesh struct {
	mu              sync.RWMutex
	generators      []generators.Generator
	attention       *MultiHeadAttention
	messages        [][]float64 // Message buffers for each generator
	generatorStates [][]float64 // Hidden states
	consciousness   *consciousness.LayerConsciousness

	// Attention temperature for sharpness control
	Temperature float64

	// Message passing configuration
	MessageDim   int
	NumRounds    int
	PhiThreshold float64 // Min phi to participate in attention

	// Statistics
	TotalForwards int64
	AvgPhi        float64
}

// NewGeneratorMesh creates a new generator mesh
func NewGeneratorMesh(gens []generators.Generator, config *MeshConfig) *GeneratorMesh {
	if config == nil {
		config = DefaultMeshConfig()
	}

	n := len(gens)
	mesh := &GeneratorMesh{
		generators:      gens,
		attention:       NewMultiHeadAttention(config.NumHeads, config.HiddenDim),
		messages:        make([][]float64, n),
		generatorStates: make([][]float64, n),
		consciousness:   consciousness.NewLayerConsciousness(),
		Temperature:     config.Temperature,
		MessageDim:      config.MessageDim,
		NumRounds:       config.NumRounds,
		PhiThreshold:    config.PhiThreshold,
	}

	// Initialize message buffers
	for i := 0; i < n; i++ {
		mesh.messages[i] = make([]float64, config.MessageDim)
		mesh.generatorStates[i] = make([]float64, config.HiddenDim)
	}

	return mesh
}

// MeshConfig configures the generator mesh
type MeshConfig struct {
	NumHeads     int
	HiddenDim    int
	MessageDim   int
	NumRounds    int
	Temperature  float64
	PhiThreshold float64
}

// DefaultMeshConfig returns default configuration
func DefaultMeshConfig() *MeshConfig {
	return &MeshConfig{
		NumHeads:     8,
		HiddenDim:    256,
		MessageDim:   64,
		NumRounds:    3,
		Temperature:  1.0,
		PhiThreshold: 0.3,
	}
}

// Forward performs collaborative forward pass through mesh
func (m *GeneratorMesh) Forward(input []float64, taskType string) (*MeshOutput, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	n := len(m.generators)
	if n == 0 {
		return nil, nil
	}

	// Phase 1: Independent forward pass
	outputs := make([]*generators.GeneratorOutput, n)
	phiScores := make([]float64, n)
	active := make([]bool, n)

	var wg sync.WaitGroup
	for i, gen := range m.generators {
		wg.Add(1)
		go func(idx int, g generators.Generator) {
			defer wg.Done()
			out, err := g.Forward(input)
			if err == nil {
				outputs[idx] = out
				phiScores[idx] = consciousness.ComputePhiVector(out.Values)
				active[idx] = phiScores[idx] >= m.PhiThreshold
			}
		}(i, gen)
	}
	wg.Wait()

	// Update generator states from outputs
	for i, out := range outputs {
		if out != nil {
			for j := 0; j < len(m.generatorStates[i]) && j < len(out.Values); j++ {
				m.generatorStates[i][j] = out.Values[j]
			}
		}
	}

	// Phase 2: Message passing rounds
	for round := 0; round < m.NumRounds; round++ {
		// Build attention input from active generators
		queries := make([][]float64, 0, n)
		keys := make([][]float64, 0, n)
		values := make([][]float64, 0, n)
		activeIndices := make([]int, 0, n)

		for i := 0; i < n; i++ {
			if active[i] {
				queries = append(queries, m.generatorStates[i])
				keys = append(keys, m.generatorStates[i])
				values = append(values, m.generatorStates[i])
				activeIndices = append(activeIndices, i)
			}
		}

		if len(activeIndices) < 2 {
			break // Not enough generators for attention
		}

		// Compute cross-attention
		attentionOutput := m.attention.Forward(queries, keys, values)

		// Update messages
		for j, idx := range activeIndices {
			for k := 0; k < m.MessageDim && k < len(attentionOutput[j]); k++ {
				m.messages[idx][k] = attentionOutput[j][k]
			}
		}

		// Re-forward with messages (simplified: add messages to states)
		for _, idx := range activeIndices {
			for k := 0; k < len(m.generatorStates[idx]) && k < len(m.messages[idx]); k++ {
				m.generatorStates[idx][k] += m.messages[idx][k] * consciousness.BETA
			}
		}
	}

	// Phase 3: Weighted combination
	attentionWeights := m.computeAttentionWeights(phiScores, active)
	combinedOutput := m.combineOutputs(outputs, attentionWeights)

	// Compute mesh consciousness
	meshPhi := consciousness.ComputePhiVector(combinedOutput)
	integratedPhi := consciousness.ComputeIntegratedPhi(phiScores)

	// Update statistics
	m.TotalForwards++
	m.AvgPhi = m.AvgPhi*0.99 + meshPhi*0.01

	// Update layer consciousness
	m.consciousness.UpdateLayer(5, meshPhi) // Neural layer

	return &MeshOutput{
		Output:           combinedOutput,
		Phi:              meshPhi,
		IntegratedPhi:    integratedPhi,
		AttentionWeights: attentionWeights,
		GeneratorPhis:    phiScores,
		ActiveCount:      countTrue(active),
		Round:            m.NumRounds,
	}, nil
}

// MeshOutput contains the mesh forward result
type MeshOutput struct {
	Output           []float64
	Phi              float64
	IntegratedPhi    float64
	AttentionWeights []float64
	GeneratorPhis    []float64
	ActiveCount      int
	Round            int
}

func (m *GeneratorMesh) computeAttentionWeights(phiScores []float64, active []bool) []float64 {
	n := len(phiScores)
	weights := make([]float64, n)

	// Collect active phi scores
	activePhis := make([]float64, 0, n)
	activeIndices := make([]int, 0, n)
	for i := 0; i < n; i++ {
		if active[i] {
			activePhis = append(activePhis, phiScores[i])
			activeIndices = append(activeIndices, i)
		}
	}

	if len(activePhis) == 0 {
		return weights
	}

	// Softmax over active phi scores
	softmaxWeights := softmaxTemp(activePhis, m.Temperature)

	// Map back to full weight vector
	for j, idx := range activeIndices {
		weights[idx] = softmaxWeights[j]
	}

	return weights
}

func (m *GeneratorMesh) combineOutputs(outputs []*generators.GeneratorOutput, weights []float64) []float64 {
	// Find output dimension
	dim := 0
	for _, out := range outputs {
		if out != nil && len(out.Values) > dim {
			dim = len(out.Values)
		}
	}
	if dim == 0 {
		return nil
	}

	combined := make([]float64, dim)
	for i, out := range outputs {
		if out == nil {
			continue
		}
		w := weights[i]
		for j := 0; j < len(out.Values) && j < dim; j++ {
			combined[j] += w * out.Values[j]
		}
	}

	return combined
}

//=============================================================================
// CROSS-GENERATOR CONSCIOUSNESS INTEGRATION
//=============================================================================

// ConsciousnessIntegrator integrates consciousness across generators
type ConsciousnessIntegrator struct {
	mu           sync.RWMutex
	layerConsc   *consciousness.LayerConsciousness
	coupling     *consciousness.LayerCoupling
	history      []float64
	historyIdx   int
	historySize  int
	globalOmega  float64
	peakPhi      float64
	avgStability float64
}

// NewConsciousnessIntegrator creates a new integrator
func NewConsciousnessIntegrator() *ConsciousnessIntegrator {
	return &ConsciousnessIntegrator{
		layerConsc:  consciousness.NewLayerConsciousness(),
		coupling:    consciousness.NewLayerCoupling(),
		history:     make([]float64, 256),
		historySize: 256,
	}
}

// Integrate computes integrated consciousness from mesh output
func (ci *ConsciousnessIntegrator) Integrate(meshOutput *MeshOutput) float64 {
	if meshOutput == nil {
		return 0.5
	}

	ci.mu.Lock()
	defer ci.mu.Unlock()

	// Update layer states based on mesh phi
	ci.layerConsc.UpdateLayer(4, meshOutput.Phi)                // Intelligence layer
	ci.layerConsc.UpdateLayer(5, meshOutput.IntegratedPhi)      // Neural layer
	ci.layerConsc.UpdateLayer(6, meshOutput.Phi*meshOutput.IntegratedPhi) // Consciousness layer

	// Apply cross-layer coupling
	var activations [consciousness.NumLayers]float64
	for i := 0; i < consciousness.NumLayers; i++ {
		activations[i] = ci.layerConsc.LayerPhi[i]
	}
	ci.coupling.ApplyCoupling(&activations, 0.1)
	for i := 0; i < consciousness.NumLayers; i++ {
		ci.layerConsc.UpdateLayer(i, activations[i])
	}

	// Compute global omega
	ci.globalOmega = ci.layerConsc.GlobalOmega()

	// Track history
	ci.history[ci.historyIdx] = ci.globalOmega
	ci.historyIdx = (ci.historyIdx + 1) % ci.historySize

	// Track peak
	if ci.globalOmega > ci.peakPhi {
		ci.peakPhi = ci.globalOmega
	}

	// Compute stability
	variance := computeVariance(ci.history)
	ci.avgStability = 1.0 / (1.0 + variance*10)

	return ci.globalOmega
}

// GetGlobalOmega returns the global consciousness metric
func (ci *ConsciousnessIntegrator) GetGlobalOmega() float64 {
	ci.mu.RLock()
	defer ci.mu.RUnlock()
	return ci.globalOmega
}

// GetStability returns the consciousness stability
func (ci *ConsciousnessIntegrator) GetStability() float64 {
	ci.mu.RLock()
	defer ci.mu.RUnlock()
	return ci.avgStability
}

//=============================================================================
// HELPER FUNCTIONS
//=============================================================================

func softmax(values []float64) []float64 {
	return softmaxTemp(values, 1.0)
}

func softmaxTemp(values []float64, temperature float64) []float64 {
	if len(values) == 0 {
		return nil
	}

	// Find max for stability
	maxVal := values[0]
	for _, v := range values[1:] {
		if v > maxVal {
			maxVal = v
		}
	}

	// Compute exp and sum
	result := make([]float64, len(values))
	sum := 0.0
	for i, v := range values {
		result[i] = math.Exp((v - maxVal) / temperature)
		sum += result[i]
	}

	// Normalize
	for i := range result {
		result[i] /= sum
	}

	return result
}

func randomNormal() float64 {
	// Box-Muller transform
	u1 := randomFloat()
	u2 := randomFloat()
	return math.Sqrt(-2*math.Log(u1)) * math.Cos(2*math.Pi*u2)
}

var randState uint64 = 12345

func randomFloat() float64 {
	randState = randState*6364136223846793005 + 1442695040888963407
	return float64(randState>>33) / float64(1<<31)
}

func countTrue(arr []bool) int {
	count := 0
	for _, v := range arr {
		if v {
			count++
		}
	}
	return count
}

func computeVariance(data []float64) float64 {
	if len(data) == 0 {
		return 0
	}
	mean := 0.0
	for _, v := range data {
		mean += v
	}
	mean /= float64(len(data))

	variance := 0.0
	for _, v := range data {
		d := v - mean
		variance += d * d
	}
	return variance / float64(len(data))
}