Source identity
| Field | Recorded value |
|---|---|
| Source type | GitHub source |
| Original record | Janady13/complete-agi-core / orchestrator/ingest-gdrive-fast.sh |
| Date | Not specified in source |
| Repository | Janady13/complete-agi-core |
| Revision | 33aa84f2aa61cedbe010240ada7f4efbdfe795a7 |
| Branch | claude/improve-math-logic-ehIW1 |
| Path or archive member | orchestrator/ingest-gdrive-fast.sh |
| Original SHA-256 | f41cb52bd0e1f3c47d20b1911dd5a47452fe2569e6497145065e64137fb13648 |
| Git blob SHA | 8c760ebe5c68df3b9ff4cb6702175383ddd11f4d |
| Reading-copy SHA-256 | 15d4859f4947fdf087de3575ebb0dc568e0df7425f0323f940fee8fd1f36b9c6 |
Recovered source transcript
Download complete source text ↓159 lines · 5.9 KB
Credential-bearing and unrelated personal details were removed from this reading copy. The source identity and original checksum remain recorded.
#!/bin/bash
# GOOGLE DRIVE FAST INGESTION - Parallel processing with strict limits
export PATH="/opt/homebrew/bin:$PATH"
GDRIVE="/Users/josephanady/Library/CloudStorage/GoogleDrive-joseph.w.anady@gmail.com"
BINARY_STORE="$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/gdrive_archive"
MB1_LEARNED="$HOME/.agi_core/learned_items.json"
LOG="$HOME/.agi_core/logs/gdrive-ingest.log"
mkdir -p "$BINARY_STORE" "$HOME/.agi_core/logs"
log() { echo "[$(date +%H:%M:%S)] $1" | tee -a "$LOG"; }
BATCH_FILE=$(mktemp)
CATALOG_FILE="$BINARY_STORE/catalog_$(date +%Y%m%d_%H%M%S).json"
echo "[]" > "$CATALOG_FILE"
log "=== GOOGLE DRIVE FAST INGESTION ==="
# Process text file and output content
process_text() {
local file="$1"
local ext="${file##*.}"
local relpath="${file#$GDRIVE/}"
local size=$(stat -f%z "$file" 2>/dev/null || echo 0)
[[ $size -gt 102400 || $size -eq 0 ]] && return # Skip >100KB or empty
local content=""
case "$ext" in
json)
content=$(head -c 2000 "$file" | jq -c '.' 2>/dev/null | head -c 1200)
[[ -n "$content" ]] && echo "[gdrive:config] $relpath: $content"
;;
sql)
content=$(grep -E "^(CREATE|ALTER|INSERT|SELECT|--)" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
[[ ${#content} -gt 30 ]] && echo "[gdrive:schema] $relpath: $content"
;;
py|js|ts)
content=$(grep -E "^(class |def |function |const |import |from |export )" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
[[ ${#content} -gt 30 ]] && echo "[gdrive:code:$ext] $relpath: $content"
;;
cpp|c|h|cs|swift)
content=$(grep -E "^(class |struct |void |int |public |private |#include|#define)" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
[[ ${#content} -gt 30 ]] && echo "[gdrive:code:$ext] $relpath: $content"
;;
sh)
content=$(grep -E "^(#|export |function )" "$file" 2>/dev/null | head -10 | tr '\n' ' ')
[[ ${#content} -gt 20 ]] && echo "[gdrive:script] $relpath: $content"
;;
yaml|yml)
content=$(head -20 "$file" | tr '\n' ' ' | head -c 800)
echo "[gdrive:config:yaml] $relpath: $content"
;;
txt|md)
content=$(head -30 "$file" | tr '\n' ' ' | head -c 1000)
[[ ${#content} -gt 30 ]] && echo "[gdrive:doc] $relpath: $content"
;;
csv)
content=$(head -3 "$file" | tr '\n' '|')
echo "[gdrive:data:csv] $relpath: $content"
;;
esac
}
export -f process_text
export GDRIVE
# Process text files with strict limits per type
log "Processing text files (limited samples)..."
for ext in json py js ts cpp c h cs swift sh yaml yml txt md csv sql; do
limit=200
[[ "$ext" == "cpp" || "$ext" == "h" ]] && limit=300
[[ "$ext" == "json" ]] && limit=400
log " Scanning .$ext files (limit $limit)..."
find "$GDRIVE" -name "*.$ext" -type f -size -100k 2>/dev/null | \
grep -v -E "(node_modules|__pycache__|\.git|build/|dist/|\.cache)" | \
head -$limit | \
while read -r file; do
process_text "$file"
done >> "$BATCH_FILE"
done
TEXT_COUNT=$(wc -l < "$BATCH_FILE" | tr -d ' ')
log "Text items extracted: $TEXT_COUNT"
# Catalog binary files (just metadata)
log "Cataloging binary files..."
BINARY_COUNT=0
for ext in zip gz jpg jpeg png gif mp4 mov pdf heic obj dll exe dmg; do
find "$GDRIVE" -name "*.$ext" -type f 2>/dev/null | \
grep -v -E "(node_modules|__pycache__|\.git)" | \
head -100 | \
while read -r file; do
relpath="${file#$GDRIVE/}"
size=$(stat -f%z "$file" 2>/dev/null || echo 0)
checksum=$(md5 -q "$file" 2>/dev/null | head -c 8 || echo "--------")
temp=$(mktemp)
jq --arg p "$relpath" --arg s "$size" --arg e "$ext" --arg c "$checksum" \
'. + [{path: $p, size: ($s|tonumber), type: $e, checksum: $c}]' \
"$CATALOG_FILE" > "$temp" && mv "$temp" "$CATALOG_FILE"
((BINARY_COUNT++))
done
done
log "Binary files cataloged: $(jq 'length' "$CATALOG_FILE")"
# Add to MB1
log "Adding to MB1 learned items..."
count=0
while IFS= read -r line; do
[[ -z "$line" ]] && continue
escaped=$(echo "$line" | jq -Rs '.')
timestamp=$(date +%s)
temp=$(mktemp)
jq --argjson c "$escaped" --argjson t "$timestamp" \
'. + [{content: $c, importance: 0.83, timestamp: $t, relevance: 0.5}]' \
"$MB1_LEARNED" > "$temp" && mv "$temp" "$MB1_LEARNED"
((count++))
[[ $((count % 200)) -eq 0 ]] && log " $count items processed..."
done < "$BATCH_FILE"
log "Ingested $count items to MB1"
# Compress
log "Compressing..."
zstd -f "$CATALOG_FILE" -o "$BINARY_STORE/catalog.zst" 2>/dev/null
zstd -f "$MB1_LEARNED" -o "$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/agi_core/learned_items.json.zst" 2>/dev/null
# Send to MB2
log "Sending to MB2..."
head -1000 "$BATCH_FILE" > /tmp/mb2_gdrive.txt
sshpass -p [credential removed] scp -o StrictHostKeyChecking=no /tmp/mb2_gdrive.txt beandaddy@192.168.4.62:/tmp/
sshpass -p [credential removed] ssh -o StrictHostKeyChecking=no -o ConnectTimeout=60 beandaddy@192.168.4.62 << 'REMOTE'
export PATH="/opt/homebrew/bin:$PATH"
LEARNED="$HOME/.seferim/data/learned_items.json"
[[ ! -f "$LEARNED" ]] && echo "[]" > "$LEARNED"
count=0
while IFS= read -r line; do
[[ -z "$line" ]] && continue
content=$(echo "$line" | jq -Rs '.')
timestamp=$(date +%s)
temp=$(mktemp)
jq --argjson c "$content" --argjson t "$timestamp" \
'. + [{content: $c, importance: 0.83, timestamp: $t, relevance: 0.5}]' \
"$LEARNED" > "$temp" && mv "$temp" "$LEARNED"
((count++))
done < /tmp/mb2_gdrive.txt
echo "MB2: $count items ingested"
rm /tmp/mb2_gdrive.txt
REMOTE
rm -f "$BATCH_FILE" /tmp/mb2_gdrive.txt
log "=== INGESTION COMPLETE ==="
log "MB1 total: $(jq 'length' "$MB1_LEARNED") items"