Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/complete-agi-core / orchestrator/ingest-gdrive.sh |
| Date | Not specified in source |
| Repository | Janady13/complete-agi-core |
| Revision | 33aa84f2aa61cedbe010240ada7f4efbdfe795a7 |
| Branch | claude/improve-math-logic-ehIW1 |
| Path or archive member | orchestrator/ingest-gdrive.sh |
| Original SHA-256 | e98d0df9998422c959b1e88c2d6525284d468c3a04af5572b2a202403db218f7 |
| Git blob SHA | a5306afed26c361d20420ff3f8d574d2f849a2a7 |
| Reading-copy SHA-256 | a94eed6bb7eec66aa40f1d3ae272c058dd3beb175494cff8a4567119db2bd592 |
Recovered source transcript
Download complete source text ↓216 lines · 8.0 KB
Credential-bearing and unrelated personal details were removed from this reading copy. The source identity and original checksum remain recorded.
#!/bin/bash
# GOOGLE DRIVE INGESTION - Smart sampling for massive codebase
export PATH="/opt/homebrew/bin:$PATH"
GDRIVE="/Users/josephanady/Library/CloudStorage/GoogleDrive-joseph.w.anady@gmail.com"
BINARY_STORE="$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/gdrive_archive"
MB1_LEARNED="$HOME/.agi_core/learned_items.json"
LOG="$HOME/.agi_core/logs/gdrive-ingest.log"
mkdir -p "$BINARY_STORE" "$HOME/.agi_core/logs"
log() { echo "[$(date +%H:%M:%S)] $1" | tee -a "$LOG"; }
# Batch add to learned items (faster than one-by-one)
BATCH_FILE=$(mktemp)
log "=== GOOGLE DRIVE INGESTION STARTING ==="
# Use Python for efficient processing
python3 << 'PYEOF'
import os
import json
import hashlib
import sys
gdrive = '/Users/josephanady/Library/CloudStorage/GoogleDrive-joseph.w.anady@gmail.com'
batch_file = os.environ.get('BATCH_FILE', '/tmp/gdrive_batch.txt')
catalog_file = '/Users/josephanady/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/gdrive_archive/catalog.json'
os.makedirs(os.path.dirname(catalog_file), exist_ok=True)
text_exts = {'json', 'txt', 'md', 'py', 'js', 'ts', 'sh', 'cpp', 'c', 'h', 'cs', 'swift', 'sql', 'yaml', 'yml', 'html', 'css', 'xml', 'csv', 'log', 'env', 'cmake'}
binary_exts = {'zip', 'gz', 'tar', 'jpg', 'jpeg', 'png', 'gif', 'mp4', 'mov', 'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'dll', 'exe', 'dmg', 'pkg', 'heic', 'obj', 'pyc', 'o', 'a', 'so', 'dylib'}
text_items = []
binary_catalog = []
seen_content = set() # Dedupe
# Sample limits per extension (don't ingest 70k cpp files)
ext_limits = {'cpp': 500, 'h': 500, 'js': 300, 'ts': 200, 'json': 500, 'sh': 200, 'py': 300, 'c': 200, 'md': 200, 'txt': 200}
ext_counts = {}
total_files = 0
text_count = 0
binary_count = 0
print("Scanning Google Drive...", file=sys.stderr)
for root, dirs, files in os.walk(gdrive):
# Skip hidden and cache dirs
dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['node_modules', '__pycache__', '.git', 'build', 'dist']]
for f in files:
if f.startswith('.'):
continue
total_files += 1
filepath = os.path.join(root, f)
relpath = filepath.replace(gdrive + '/', '')
ext = f.split('.')[-1].lower() if '.' in f else 'noext'
try:
size = os.path.getsize(filepath)
except:
continue
# Binary files - just catalog
if ext in binary_exts or size > 1024*1024: # >1MB = binary
binary_catalog.append({
'path': relpath,
'size': size,
'type': ext
})
binary_count += 1
continue
# Text files - check limits
if ext in text_exts:
ext_counts[ext] = ext_counts.get(ext, 0) + 1
limit = ext_limits.get(ext, 100)
if ext_counts[ext] > limit:
continue # Skip, already have enough of this type
# Read and extract content
try:
with open(filepath, 'r', errors='ignore') as fh:
content = fh.read(3000) # First 3KB
if len(content) < 50:
continue
# Dedupe by content hash
content_hash = hashlib.md5(content.encode()).hexdigest()[:12]
if content_hash in seen_content:
continue
seen_content.add(content_hash)
# Extract meaningful parts based on type
if ext in ['cpp', 'c', 'h', 'cs', 'swift']:
lines = [l.strip() for l in content.split('\n') if l.strip() and (
l.strip().startswith(('class ', 'struct ', 'void ', 'int ', 'public ', 'private ', 'def ', 'func ', '#include', '#define', '//'))
)][:20]
if lines:
content = ' '.join(lines)
elif ext in ['py', 'js', 'ts']:
lines = [l.strip() for l in content.split('\n') if l.strip() and (
l.strip().startswith(('class ', 'def ', 'function ', 'const ', 'import ', 'from ', 'export ', '#'))
)][:20]
if lines:
content = ' '.join(lines)
elif ext == 'sh':
lines = [l.strip() for l in content.split('\n') if l.strip() and (
l.strip().startswith(('#', 'export ', 'function ')) or '=' in l
)][:15]
if lines:
content = ' '.join(lines)
elif ext == 'json':
try:
content = json.dumps(json.loads(content))[:1500]
except:
content = content[:1000].replace('\n', ' ')
elif ext == 'sql':
lines = [l for l in content.split('\n') if any(kw in l.upper() for kw in ['CREATE', 'ALTER', 'INSERT', 'SELECT', '--'])][:20]
content = ' '.join(lines) if lines else content[:1000]
else:
content = content[:1500].replace('\n', ' ')
if len(content) > 50:
tag = f"[gdrive:code:{ext}]" if ext in ['cpp','c','h','py','js','ts','cs','swift'] else f"[gdrive:{ext}]"
text_items.append(f"{tag} {relpath}: {content[:1500]}")
text_count += 1
except Exception as e:
continue
print(f"Total files scanned: {total_files}", file=sys.stderr)
print(f"Text items extracted: {text_count}", file=sys.stderr)
print(f"Binary files cataloged: {binary_count}", file=sys.stderr)
# Write batch file
with open(batch_file, 'w') as f:
for item in text_items:
f.write(item + '\n')
# Write catalog
with open(catalog_file, 'w') as f:
json.dump(binary_catalog, f)
print(f"BATCH={batch_file}")
print(f"CATALOG={catalog_file}")
print(f"TEXT_COUNT={text_count}")
print(f"BINARY_COUNT={binary_count}")
PYEOF
# Get output values
BATCH_FILE="/tmp/gdrive_batch.txt"
CATALOG_FILE="$BINARY_STORE/catalog.json"
TEXT_COUNT=$(wc -l < "$BATCH_FILE" | tr -d ' ')
log "Text items to ingest: $TEXT_COUNT"
# Add to MB1 learned items in batches
log "Adding to MB1 learned items..."
count=0
while IFS= read -r line; do
[[ -z "$line" ]] && continue
escaped=$(echo "$line" | jq -Rs '.')
timestamp=$(date +%s)
temp=$(mktemp)
jq --argjson c "$escaped" --argjson t "$timestamp" \
'. + [{content: $c, importance: 0.83, timestamp: $t, relevance: 0.5}]' \
"$MB1_LEARNED" > "$temp" && mv "$temp" "$MB1_LEARNED"
((count++))
[[ $((count % 100)) -eq 0 ]] && log " Processed $count items..."
done < "$BATCH_FILE"
log "Ingested $count items to MB1"
# Compress catalog
log "Compressing catalog..."
zstd -f "$CATALOG_FILE" -o "$BINARY_STORE/catalog.zst" 2>/dev/null
# Compress learned_items for iCloud
log "Compressing learned_items for iCloud..."
zstd -f "$MB1_LEARNED" -o "$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/agi_core/learned_items.json.zst" 2>/dev/null
# Send sample to MB2 (limit for SSH)
log "Sending to MB2..."
head -1500 "$BATCH_FILE" > /tmp/mb2_batch.txt
sshpass -p [credential removed] ssh -o StrictHostKeyChecking=no -o ConnectTimeout=60 beandaddy@192.168.4.62 'cat > /tmp/gdrive_ingest.txt' < /tmp/mb2_batch.txt
sshpass -p [credential removed] ssh -o StrictHostKeyChecking=no -o ConnectTimeout=60 beandaddy@192.168.4.62 << 'REMOTE'
export PATH="/opt/homebrew/bin:$PATH"
LEARNED="$HOME/.seferim/data/learned_items.json"
[[ ! -f "$LEARNED" ]] && echo "[]" > "$LEARNED"
count=0
while IFS= read -r line; do
[[ -z "$line" ]] && continue
content=$(echo "$line" | jq -Rs '.')
timestamp=$(date +%s)
temp=$(mktemp)
jq --argjson c "$content" --argjson t "$timestamp" \
'. + [{content: $c, importance: 0.83, timestamp: $t, relevance: 0.5}]' \
"$LEARNED" > "$temp" && mv "$temp" "$LEARNED"
((count++))
done < /tmp/gdrive_ingest.txt
echo "MB2: Ingested $count items"
rm /tmp/gdrive_ingest.txt
REMOTE
rm -f "$BATCH_FILE" /tmp/mb2_batch.txt
log "=== INGESTION COMPLETE ==="
log "MB1 total items: $(jq 'length' "$MB1_LEARNED")"