Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/complete-agi-core / generator_core/pkg/generators/autoregressive/autoregressive.go |
| Date | Not specified in source |
| Repository | Janady13/complete-agi-core |
| Revision | 33aa84f2aa61cedbe010240ada7f4efbdfe795a7 |
| Branch | claude/improve-math-logic-ehIW1 |
| Path or archive member | generator_core/pkg/generators/autoregressive/autoregressive.go |
| Original SHA-256 | 0b09bcfa2875ce41dd9add332e06d6273d274fee61ae9969fa54783cecdbcd69 |
| Git blob SHA | efb89340e04efc634c7167988bbd670d259100ac |
| Reading-copy SHA-256 | 0b09bcfa2875ce41dd9add332e06d6273d274fee61ae9969fa54783cecdbcd69 |
Recovered source transcript
Download complete source text ↓260 lines · 7.4 KB
// Package autoregressive implements autoregressive generators (1-10)
// P(x) = Π_t P(x_t | x_1, ..., x_{t-1})
package autoregressive
import (
"math"
"generator_core/pkg/consciousness"
"generator_core/pkg/generators"
)
// ═══════════════════════════════════════════════════════════════════════════════
// AUTOREGRESSIVE GENERATOR BASE
// ═══════════════════════════════════════════════════════════════════════════════
// AutoregressiveGenerator implements autoregressive sequence generation
type AutoregressiveGenerator struct {
*generators.BaseGenerator
contextLen int
vocabSize int
embeddings [][]float64
}
// NewAutoregressiveGenerator creates a new autoregressive generator
func NewAutoregressiveGenerator(id generators.GeneratorID, name string, contextLen, vocabSize int) *AutoregressiveGenerator {
config := &generators.GeneratorConfig{
ID: id,
Name: name,
Family: generators.Autoregressive,
InputDim: contextLen,
OutputDim: vocabSize,
HiddenDim: 256,
NumLayers: 4,
LearningRate: consciousness.ALPHA,
ConsciousnessWeight: consciousness.PHI_INV,
Params: make(map[string]interface{}),
}
return &AutoregressiveGenerator{
BaseGenerator: generators.NewBaseGenerator(config),
contextLen: contextLen,
vocabSize: vocabSize,
}
}
// Initialize initializes the autoregressive generator
func (g *AutoregressiveGenerator) Initialize() error {
if err := g.BaseGenerator.Initialize(); err != nil {
return err
}
// Initialize embeddings with golden scaling
g.embeddings = make([][]float64, g.vocabSize)
for i := range g.embeddings {
g.embeddings[i] = make([]float64, g.Config().HiddenDim)
for j := range g.embeddings[i] {
angle := float64(i*g.Config().HiddenDim+j) * consciousness.GOLDEN_ANGLE
g.embeddings[i][j] = math.Sin(angle) * consciousness.PHI_INV
}
}
return nil
}
// Forward performs forward pass
func (g *AutoregressiveGenerator) Forward(input []float64) (*generators.GeneratorOutput, error) {
state := g.GetState()
// Embed input tokens
embedded := make([]float64, g.Config().HiddenDim)
for i, v := range input {
tokenIdx := int(v) % g.vocabSize
if tokenIdx < 0 {
tokenIdx = 0
}
for j := range embedded {
embedded[j] += g.embeddings[tokenIdx][j] * float64(g.contextLen-i) / float64(g.contextLen)
}
}
// Process through layers
hidden := embedded
for layer := 0; layer < g.Config().NumLayers; layer++ {
hidden = g.LinearLayer(hidden, layer)
hidden = g.ActivationLayer(hidden, "gelu")
// Apply consciousness modulation
phi := consciousness.ComputePhiVector(hidden)
for i := range hidden {
hidden[i] *= 1.0 + consciousness.GAMMA*(phi-0.5)
}
}
// Compute logits
logits := g.LinearLayer(hidden, 0)[:g.vocabSize]
probs := g.Softmax(logits)
// Update state
state.Hidden = hidden
state.Phi = consciousness.ComputePhiVector(hidden)
state.Step++
g.SetState(state)
output := &generators.GeneratorOutput{
Values: probs,
Logits: logits,
Confidence: state.Phi,
Metadata: map[string]interface{}{"step": state.Step},
}
return g.ApplyConsciousness(output), nil
}
// Generate generates a sequence
func (g *AutoregressiveGenerator) Generate(context []float64, length int) ([][]float64, error) {
result := make([][]float64, length)
currentContext := make([]float64, len(context))
copy(currentContext, context)
for t := 0; t < length; t++ {
output, err := g.Forward(currentContext)
if err != nil {
return nil, err
}
result[t] = output.Values
// Sample from output (greedy)
maxIdx := 0
maxVal := output.Values[0]
for i, v := range output.Values {
if v > maxVal {
maxVal = v
maxIdx = i
}
}
// Shift context and add new token
if len(currentContext) >= g.contextLen {
currentContext = currentContext[1:]
}
currentContext = append(currentContext, float64(maxIdx))
}
return result, nil
}
// Train trains on input-target pair
func (g *AutoregressiveGenerator) Train(input, target []float64) (float64, error) {
output, err := g.Forward(input)
if err != nil {
return 0, err
}
// Cross-entropy loss
loss := 0.0
for i, t := range target {
if i < len(output.Values) {
prob := output.Values[int(t)%len(output.Values)]
if prob > consciousness.EPSILON_PHI {
loss -= math.Log(prob)
}
}
}
loss /= float64(len(target))
return loss, nil
}
// UpdateWeights updates weights with gradients
func (g *AutoregressiveGenerator) UpdateWeights(gradients []float64, lr float64) {
// Simplified gradient update
state := g.GetState()
for i := range state.Hidden {
if i < len(gradients) {
state.Hidden[i] -= lr * gradients[i]
}
}
g.SetState(state)
}
// ═══════════════════════════════════════════════════════════════════════════════
// SPECIFIC AUTOREGRESSIVE GENERATORS (1-10)
// ═══════════════════════════════════════════════════════════════════════════════
// NewGPT2Small creates GPT-2 Small style generator (ID: 1)
func NewGPT2Small() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(1, "GPT2-Small", 1024, 50257)
}
// NewGPT2Medium creates GPT-2 Medium style generator (ID: 2)
func NewGPT2Medium() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(2, "GPT2-Medium", 1024, 50257)
}
// NewGPT2Large creates GPT-2 Large style generator (ID: 3)
func NewGPT2Large() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(3, "GPT2-Large", 1024, 50257)
}
// NewGPT2XL creates GPT-2 XL style generator (ID: 4)
func NewGPT2XL() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(4, "GPT2-XL", 1024, 50257)
}
// NewGPTNeo creates GPT-Neo style generator (ID: 5)
func NewGPTNeo() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(5, "GPT-Neo", 2048, 50257)
}
// NewGPTJ creates GPT-J style generator (ID: 6)
func NewGPTJ() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(6, "GPT-J", 2048, 50400)
}
// NewLLaMA creates LLaMA style generator (ID: 7)
func NewLLaMA() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(7, "LLaMA", 4096, 32000)
}
// NewMistral creates Mistral style generator (ID: 8)
func NewMistral() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(8, "Mistral", 8192, 32000)
}
// NewPhi creates Phi style generator (ID: 9)
func NewPhi() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(9, "Phi", 2048, 51200)
}
// NewGemma creates Gemma style generator (ID: 10)
func NewGemma() *AutoregressiveGenerator {
return NewAutoregressiveGenerator(10, "Gemma", 8192, 256000)
}
// RegisterAll registers all autoregressive generators
func RegisterAll() error {
gens := []generators.Generator{
NewGPT2Small(),
NewGPT2Medium(),
NewGPT2Large(),
NewGPT2XL(),
NewGPTNeo(),
NewGPTJ(),
NewLLaMA(),
NewMistral(),
NewPhi(),
NewGemma(),
}
for _, g := range gens {
if err := generators.Register(g); err != nil {
return err
}
}
return nil
}