Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/complete-agi-core / seferim/knowledge_graph.go |
| Date | Not specified in source |
| Repository | Janady13/complete-agi-core |
| Revision | 33aa84f2aa61cedbe010240ada7f4efbdfe795a7 |
| Branch | claude/improve-math-logic-ehIW1 |
| Path or archive member | seferim/knowledge_graph.go |
| Original SHA-256 | 90860ced45d517f051fb6062dd8682852baa0d3cf20547120c72ce7a847ae0a0 |
| Git blob SHA | 9db5285305341a00e54ec182c1f01228306db6c9 |
| Reading-copy SHA-256 | 90860ced45d517f051fb6062dd8682852baa0d3cf20547120c72ce7a847ae0a0 |
Recovered source transcript
Download complete source text ↓803 lines · 20.3 KB
// Package main implements a knowledge graph with semantic relations.
//
// Features:
// - Graph-based knowledge storage with typed edges
// - Semantic similarity via embeddings
// - Graph traversal and querying
// - Active learning with information gain
// - Spaced repetition scheduling
package main
import (
"database/sql"
"encoding/json"
"fmt"
"math"
"os"
"sort"
"sync"
"time"
_ "github.com/mattn/go-sqlite3"
)
//=============================================================================
// CONSTANTS
//=============================================================================
const (
PHI = 1.6180339887498948482
EmbeddingDim = 256
MaxGraphDepth = 5
MinConfidence = 0.1
DecayRate = 0.02 * PHI
ReinforceFactor = 0.1 * PHI
)
// Relation types
const (
RelIsA = "is_a"
RelPartOf = "part_of"
RelRelatedTo = "related_to"
RelCauses = "causes"
RelRequires = "requires"
RelSimilarTo = "similar_to"
RelOppositeOf = "opposite_of"
RelExampleOf = "example_of"
RelUsedFor = "used_for"
RelCreatedBy = "created_by"
)
//=============================================================================
// DATA STRUCTURES
//=============================================================================
// Node represents a knowledge graph node
type Node struct {
ID int64 `json:"id"`
Concept string `json:"concept"`
Domain string `json:"domain"`
Category string `json:"category"`
Confidence float64 `json:"confidence"`
Importance float64 `json:"importance"`
AccessCount int64 `json:"access_count"`
Embedding []float32 `json:"embedding,omitempty"`
CreatedAt time.Time `json:"created_at"`
LastAccessed time.Time `json:"last_accessed"`
LastReinforced time.Time `json:"last_reinforced,omitempty"`
}
// Edge represents a relation between nodes
type Edge struct {
ID int64 `json:"id"`
SourceID int64 `json:"source_id"`
TargetID int64 `json:"target_id"`
Relation string `json:"relation"`
Strength float64 `json:"strength"`
EvidenceCount int `json:"evidence_count"`
}
// RepetitionState tracks spaced repetition for a concept
type RepetitionState struct {
ConceptID int64 `json:"concept_id"`
RepetitionNumber int `json:"repetition_number"`
EasinessFactor float64 `json:"easiness_factor"`
IntervalDays float64 `json:"interval_days"`
NextReview time.Time `json:"next_review"`
LastReview time.Time `json:"last_review"`
LastQuality int `json:"last_quality"` // 0-5
}
// QueryResult contains graph query results
type QueryResult struct {
Nodes []*Node `json:"nodes"`
Edges []*Edge `json:"edges"`
QueryTimeMs float64 `json:"query_time_ms"`
TotalNodes int `json:"total_nodes"`
}
//=============================================================================
// KNOWLEDGE GRAPH
//=============================================================================
// KnowledgeGraph is the main graph structure
type KnowledgeGraph struct {
mu sync.RWMutex
db *sql.DB
dbPath string
// In-memory caches
nodeCache map[int64]*Node
conceptToID map[string]int64
adjacency map[int64][]int64 // node -> connected nodes
edgeCache map[int64][]*Edge
// Statistics
totalNodes int64
totalEdges int64
avgConfidence float64
}
// NewKnowledgeGraph creates a new knowledge graph
func NewKnowledgeGraph(dbPath string) (*KnowledgeGraph, error) {
kg := &KnowledgeGraph{
dbPath: dbPath,
nodeCache: make(map[int64]*Node),
conceptToID: make(map[string]int64),
adjacency: make(map[int64][]int64),
edgeCache: make(map[int64][]*Edge),
}
// Open database
db, err := sql.Open("sqlite3", dbPath)
if err != nil {
return nil, err
}
kg.db = db
// Initialize schema
if err := kg.initSchema(); err != nil {
return nil, err
}
// Load caches
if err := kg.loadCaches(); err != nil {
return nil, err
}
return kg, nil
}
func (kg *KnowledgeGraph) initSchema() error {
schema := `
CREATE TABLE IF NOT EXISTS nodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
concept TEXT UNIQUE NOT NULL,
domain TEXT,
category TEXT,
confidence REAL DEFAULT 0.5,
importance REAL DEFAULT 0.5,
access_count INTEGER DEFAULT 0,
embedding BLOB,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
last_accessed DATETIME DEFAULT CURRENT_TIMESTAMP,
last_reinforced DATETIME
);
CREATE TABLE IF NOT EXISTS edges (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_id INTEGER NOT NULL,
target_id INTEGER NOT NULL,
relation TEXT NOT NULL,
strength REAL DEFAULT 0.5,
evidence_count INTEGER DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (source_id) REFERENCES nodes(id),
FOREIGN KEY (target_id) REFERENCES nodes(id),
UNIQUE(source_id, target_id, relation)
);
CREATE TABLE IF NOT EXISTS repetition (
id INTEGER PRIMARY KEY AUTOINCREMENT,
concept_id INTEGER UNIQUE NOT NULL,
repetition_number INTEGER DEFAULT 0,
easiness_factor REAL DEFAULT 2.5,
interval_days REAL DEFAULT 1.0,
next_review DATETIME,
last_review DATETIME,
last_quality INTEGER DEFAULT 3,
FOREIGN KEY (concept_id) REFERENCES nodes(id)
);
CREATE INDEX IF NOT EXISTS idx_nodes_concept ON nodes(concept);
CREATE INDEX IF NOT EXISTS idx_nodes_domain ON nodes(domain);
CREATE INDEX IF NOT EXISTS idx_nodes_confidence ON nodes(confidence);
CREATE INDEX IF NOT EXISTS idx_edges_source ON edges(source_id);
CREATE INDEX IF NOT EXISTS idx_edges_target ON edges(target_id);
CREATE INDEX IF NOT EXISTS idx_edges_relation ON edges(relation);
CREATE INDEX IF NOT EXISTS idx_repetition_next ON repetition(next_review);
`
_, err := kg.db.Exec(schema)
return err
}
func (kg *KnowledgeGraph) loadCaches() error {
// Load nodes
rows, err := kg.db.Query(`
SELECT id, concept, domain, category, confidence, importance, access_count
FROM nodes
`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
node := &Node{}
err := rows.Scan(&node.ID, &node.Concept, &node.Domain, &node.Category,
&node.Confidence, &node.Importance, &node.AccessCount)
if err != nil {
continue
}
kg.nodeCache[node.ID] = node
kg.conceptToID[node.Concept] = node.ID
kg.totalNodes++
kg.avgConfidence += node.Confidence
}
if kg.totalNodes > 0 {
kg.avgConfidence /= float64(kg.totalNodes)
}
// Load edges
edgeRows, err := kg.db.Query(`
SELECT id, source_id, target_id, relation, strength, evidence_count
FROM edges
`)
if err != nil {
return err
}
defer edgeRows.Close()
for edgeRows.Next() {
edge := &Edge{}
err := edgeRows.Scan(&edge.ID, &edge.SourceID, &edge.TargetID,
&edge.Relation, &edge.Strength, &edge.EvidenceCount)
if err != nil {
continue
}
kg.edgeCache[edge.SourceID] = append(kg.edgeCache[edge.SourceID], edge)
kg.adjacency[edge.SourceID] = append(kg.adjacency[edge.SourceID], edge.TargetID)
kg.totalEdges++
}
return nil
}
// Close closes the database connection
func (kg *KnowledgeGraph) Close() error {
return kg.db.Close()
}
//=============================================================================
// NODE OPERATIONS
//=============================================================================
// AddNode adds or updates a node in the graph
func (kg *KnowledgeGraph) AddNode(node *Node) (int64, error) {
kg.mu.Lock()
defer kg.mu.Unlock()
// Check if exists
if existingID, ok := kg.conceptToID[node.Concept]; ok {
// Update existing
_, err := kg.db.Exec(`
UPDATE nodes SET
domain = COALESCE(?, domain),
category = COALESCE(?, category),
confidence = ?,
importance = ?,
embedding = ?,
last_accessed = CURRENT_TIMESTAMP
WHERE id = ?
`, node.Domain, node.Category, node.Confidence, node.Importance,
embeddingToBytes(node.Embedding), existingID)
if err != nil {
return 0, err
}
kg.nodeCache[existingID] = node
node.ID = existingID
return existingID, nil
}
// Insert new
result, err := kg.db.Exec(`
INSERT INTO nodes (concept, domain, category, confidence, importance, embedding)
VALUES (?, ?, ?, ?, ?, ?)
`, node.Concept, node.Domain, node.Category, node.Confidence, node.Importance,
embeddingToBytes(node.Embedding))
if err != nil {
return 0, err
}
id, _ := result.LastInsertId()
node.ID = id
node.CreatedAt = time.Now()
node.LastAccessed = time.Now()
kg.nodeCache[id] = node
kg.conceptToID[node.Concept] = id
kg.totalNodes++
// Initialize repetition schedule
kg.initRepetition(id)
return id, nil
}
// GetNode retrieves a node by concept name
func (kg *KnowledgeGraph) GetNode(concept string) (*Node, error) {
kg.mu.RLock()
id, ok := kg.conceptToID[concept]
if ok {
node := kg.nodeCache[id]
kg.mu.RUnlock()
kg.updateAccess(id)
return node, nil
}
kg.mu.RUnlock()
// Query database
row := kg.db.QueryRow(`
SELECT id, concept, domain, category, confidence, importance,
access_count, embedding, created_at, last_accessed
FROM nodes WHERE concept = ?
`, concept)
node := &Node{}
var embedding []byte
err := row.Scan(&node.ID, &node.Concept, &node.Domain, &node.Category,
&node.Confidence, &node.Importance, &node.AccessCount,
&embedding, &node.CreatedAt, &node.LastAccessed)
if err != nil {
return nil, err
}
node.Embedding = bytesToEmbedding(embedding)
kg.mu.Lock()
kg.nodeCache[node.ID] = node
kg.conceptToID[node.Concept] = node.ID
kg.mu.Unlock()
kg.updateAccess(node.ID)
return node, nil
}
func (kg *KnowledgeGraph) updateAccess(nodeID int64) {
kg.db.Exec(`
UPDATE nodes SET
access_count = access_count + 1,
last_accessed = CURRENT_TIMESTAMP
WHERE id = ?
`, nodeID)
}
//=============================================================================
// EDGE OPERATIONS
//=============================================================================
// AddEdge adds or strengthens an edge between nodes
func (kg *KnowledgeGraph) AddEdge(sourceID, targetID int64, relation string, strength float64) error {
kg.mu.Lock()
defer kg.mu.Unlock()
// Try update first
result, err := kg.db.Exec(`
UPDATE edges SET
strength = MIN(1.0, strength + ?),
evidence_count = evidence_count + 1
WHERE source_id = ? AND target_id = ? AND relation = ?
`, strength*0.1, sourceID, targetID, relation)
if err != nil {
return err
}
rowsAffected, _ := result.RowsAffected()
if rowsAffected > 0 {
return nil
}
// Insert new edge
_, err = kg.db.Exec(`
INSERT INTO edges (source_id, target_id, relation, strength)
VALUES (?, ?, ?, ?)
`, sourceID, targetID, relation, strength)
if err != nil {
return err
}
// Update caches
edge := &Edge{
SourceID: sourceID,
TargetID: targetID,
Relation: relation,
Strength: strength,
EvidenceCount: 1,
}
kg.edgeCache[sourceID] = append(kg.edgeCache[sourceID], edge)
kg.adjacency[sourceID] = append(kg.adjacency[sourceID], targetID)
kg.totalEdges++
return nil
}
// GetEdgesFrom returns all edges from a node
func (kg *KnowledgeGraph) GetEdgesFrom(nodeID int64) []*Edge {
kg.mu.RLock()
defer kg.mu.RUnlock()
return kg.edgeCache[nodeID]
}
//=============================================================================
// GRAPH TRAVERSAL
//=============================================================================
// Traverse performs BFS traversal from a starting node
func (kg *KnowledgeGraph) Traverse(startConcept string, maxDepth int, minConfidence float64) *QueryResult {
start := time.Now()
node, err := kg.GetNode(startConcept)
if err != nil {
return &QueryResult{QueryTimeMs: float64(time.Since(start).Microseconds()) / 1000}
}
kg.mu.RLock()
defer kg.mu.RUnlock()
visited := make(map[int64]bool)
nodes := []*Node{node}
edges := []*Edge{}
visited[node.ID] = true
// BFS
queue := []struct {
id int64
depth int
}{{node.ID, 0}}
for len(queue) > 0 {
current := queue[0]
queue = queue[1:]
if current.depth >= maxDepth {
continue
}
for _, edge := range kg.edgeCache[current.id] {
if visited[edge.TargetID] {
continue
}
targetNode := kg.nodeCache[edge.TargetID]
if targetNode == nil || targetNode.Confidence < minConfidence {
continue
}
visited[edge.TargetID] = true
nodes = append(nodes, targetNode)
edges = append(edges, edge)
queue = append(queue, struct {
id int64
depth int
}{edge.TargetID, current.depth + 1})
}
}
return &QueryResult{
Nodes: nodes,
Edges: edges,
QueryTimeMs: float64(time.Since(start).Microseconds()) / 1000,
TotalNodes: len(nodes),
}
}
// FindSimilar finds nodes similar to the given embedding
func (kg *KnowledgeGraph) FindSimilar(embedding []float32, topK int, minSimilarity float64) []*Node {
kg.mu.RLock()
defer kg.mu.RUnlock()
type scoredNode struct {
node *Node
score float64
}
var scored []scoredNode
for _, node := range kg.nodeCache {
if len(node.Embedding) == 0 {
continue
}
sim := cosineSimilarity(embedding, node.Embedding)
if sim >= minSimilarity {
scored = append(scored, scoredNode{node, sim})
}
}
// Sort by similarity
sort.Slice(scored, func(i, j int) bool {
return scored[i].score > scored[j].score
})
// Return top K
result := make([]*Node, 0, topK)
for i := 0; i < len(scored) && i < topK; i++ {
result = append(result, scored[i].node)
}
return result
}
//=============================================================================
// ACTIVE LEARNING
//=============================================================================
// ComputeInformationGain estimates information gain for learning a concept
func (kg *KnowledgeGraph) ComputeInformationGain(concept string, domain string) float64 {
kg.mu.RLock()
defer kg.mu.RUnlock()
// If concept exists, less to gain
if _, ok := kg.conceptToID[concept]; ok {
return 0.1 // Small gain for reinforcement
}
// Estimate based on domain connectivity
domainNodes := 0
domainConfidence := 0.0
for _, node := range kg.nodeCache {
if node.Domain == domain {
domainNodes++
domainConfidence += node.Confidence
}
}
if domainNodes == 0 {
return 1.0 // New domain, high gain
}
avgDomainConfidence := domainConfidence / float64(domainNodes)
// Information gain = entropy reduction
// More uncertain domains (lower avg confidence) have higher gain
gain := 1.0 - avgDomainConfidence
// Bonus for connecting to existing knowledge
gain *= (1.0 + math.Log(float64(domainNodes+1))/10.0)
return math.Min(1.0, gain)
}
// SelectNextToLearn selects the next concept to learn for maximum information gain
func (kg *KnowledgeGraph) SelectNextToLearn(candidates []string, domains []string) (string, float64) {
if len(candidates) == 0 {
return "", 0
}
bestConcept := candidates[0]
bestGain := 0.0
for i, concept := range candidates {
domain := ""
if i < len(domains) {
domain = domains[i]
}
gain := kg.ComputeInformationGain(concept, domain)
if gain > bestGain {
bestGain = gain
bestConcept = concept
}
}
return bestConcept, bestGain
}
//=============================================================================
// SPACED REPETITION (SM-2 ALGORITHM)
//=============================================================================
func (kg *KnowledgeGraph) initRepetition(conceptID int64) {
kg.db.Exec(`
INSERT OR IGNORE INTO repetition (concept_id, next_review, last_review)
VALUES (?, datetime('now', '+1 day'), datetime('now'))
`, conceptID)
}
// GetDueForReview returns concepts due for review
func (kg *KnowledgeGraph) GetDueForReview(limit int) []*Node {
rows, err := kg.db.Query(`
SELECT n.id, n.concept, n.domain, n.category, n.confidence
FROM nodes n
JOIN repetition r ON n.id = r.concept_id
WHERE r.next_review <= datetime('now')
ORDER BY r.next_review ASC
LIMIT ?
`, limit)
if err != nil {
return nil
}
defer rows.Close()
var nodes []*Node
for rows.Next() {
node := &Node{}
rows.Scan(&node.ID, &node.Concept, &node.Domain, &node.Category, &node.Confidence)
nodes = append(nodes, node)
}
return nodes
}
// RecordReview records a review result and updates the schedule
func (kg *KnowledgeGraph) RecordReview(conceptID int64, quality int) error {
if quality < 0 {
quality = 0
}
if quality > 5 {
quality = 5
}
// Get current state
var rep RepetitionState
row := kg.db.QueryRow(`
SELECT repetition_number, easiness_factor, interval_days
FROM repetition WHERE concept_id = ?
`, conceptID)
err := row.Scan(&rep.RepetitionNumber, &rep.EasinessFactor, &rep.IntervalDays)
if err != nil {
return err
}
// SM-2 algorithm
if quality < 3 {
// Failed review, reset
rep.RepetitionNumber = 0
rep.IntervalDays = 1
} else {
// Successful review
rep.RepetitionNumber++
if rep.RepetitionNumber == 1 {
rep.IntervalDays = 1
} else if rep.RepetitionNumber == 2 {
rep.IntervalDays = 6
} else {
rep.IntervalDays = rep.IntervalDays * rep.EasinessFactor
}
}
// Update easiness factor
rep.EasinessFactor = rep.EasinessFactor + (0.1 - float64(5-quality)*(0.08+float64(5-quality)*0.02))
if rep.EasinessFactor < 1.3 {
rep.EasinessFactor = 1.3
}
// Update database
_, err = kg.db.Exec(`
UPDATE repetition SET
repetition_number = ?,
easiness_factor = ?,
interval_days = ?,
next_review = datetime('now', '+' || ? || ' days'),
last_review = datetime('now'),
last_quality = ?
WHERE concept_id = ?
`, rep.RepetitionNumber, rep.EasinessFactor, rep.IntervalDays,
int(rep.IntervalDays), quality, conceptID)
// Reinforce node confidence
if quality >= 3 {
kg.reinforceNode(conceptID, float64(quality)/5.0)
} else {
kg.decayNode(conceptID)
}
return err
}
func (kg *KnowledgeGraph) reinforceNode(nodeID int64, factor float64) {
kg.db.Exec(`
UPDATE nodes SET
confidence = MIN(1.0, confidence + ? * (1.0 - confidence)),
last_reinforced = datetime('now')
WHERE id = ?
`, ReinforceFactor*factor, nodeID)
}
func (kg *KnowledgeGraph) decayNode(nodeID int64) {
kg.db.Exec(`
UPDATE nodes SET
confidence = MAX(0.1, confidence * (1.0 - ?))
WHERE id = ?
`, DecayRate, nodeID)
}
//=============================================================================
// STATISTICS
//=============================================================================
// Stats returns graph statistics
func (kg *KnowledgeGraph) Stats() map[string]interface{} {
kg.mu.RLock()
defer kg.mu.RUnlock()
// Domain distribution
domainCounts := make(map[string]int)
for _, node := range kg.nodeCache {
domainCounts[node.Domain]++
}
// Relation distribution
relationCounts := make(map[string]int)
for _, edges := range kg.edgeCache {
for _, edge := range edges {
relationCounts[edge.Relation]++
}
}
return map[string]interface{}{
"total_nodes": kg.totalNodes,
"total_edges": kg.totalEdges,
"avg_confidence": kg.avgConfidence,
"domains": domainCounts,
"relations": relationCounts,
}
}
//=============================================================================
// HELPER FUNCTIONS
//=============================================================================
func embeddingToBytes(embedding []float32) []byte {
if len(embedding) == 0 {
return nil
}
bytes := make([]byte, len(embedding)*4)
for i, v := range embedding {
bits := math.Float32bits(v)
bytes[i*4] = byte(bits)
bytes[i*4+1] = byte(bits >> 8)
bytes[i*4+2] = byte(bits >> 16)
bytes[i*4+3] = byte(bits >> 24)
}
return bytes
}
func bytesToEmbedding(bytes []byte) []float32 {
if len(bytes) == 0 {
return nil
}
embedding := make([]float32, len(bytes)/4)
for i := 0; i < len(embedding); i++ {
bits := uint32(bytes[i*4]) | uint32(bytes[i*4+1])<<8 |
uint32(bytes[i*4+2])<<16 | uint32(bytes[i*4+3])<<24
embedding[i] = math.Float32frombits(bits)
}
return embedding
}
func cosineSimilarity(a, b []float32) float64 {
if len(a) != len(b) || len(a) == 0 {
return 0
}
var dot, normA, normB float64
for i := range a {
dot += float64(a[i]) * float64(b[i])
normA += float64(a[i]) * float64(a[i])
normB += float64(b[i]) * float64(b[i])
}
if normA == 0 || normB == 0 {
return 0
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}
//=============================================================================
// MAIN (CLI)
//=============================================================================
func main() {
dbPath := os.Getenv("KNOWLEDGE_DB")
if dbPath == "" {
dbPath = "knowledge_graph.db"
}
kg, err := NewKnowledgeGraph(dbPath)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to open knowledge graph: %v\n", err)
os.Exit(1)
}
defer kg.Close()
// Print stats
stats := kg.Stats()
statsJSON, _ := json.MarshalIndent(stats, "", " ")
fmt.Println(string(statsJSON))
}