#!/bin/bash
# GOOGLE DRIVE FAST INGESTION - Parallel processing with strict limits
export PATH="/opt/homebrew/bin:$PATH"

GDRIVE="/Users/josephanady/Library/CloudStorage/GoogleDrive-joseph.w.anady@gmail.com"
BINARY_STORE="$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/gdrive_archive"
MB1_LEARNED="$HOME/.thataiguy/learned_items.json"
LOG="$HOME/.thataiguy/logs/gdrive-ingest.log"

mkdir -p "$BINARY_STORE" "$HOME/.thataiguy/logs"

log() { echo "[$(date +%H:%M:%S)] $1" | tee -a "$LOG"; }

BATCH_FILE=$(mktemp)
CATALOG_FILE="$BINARY_STORE/catalog_$(date +%Y%m%d_%H%M%S).json"
echo "[]" > "$CATALOG_FILE"

log "=== GOOGLE DRIVE FAST INGESTION ==="

# Process text file and output content
process_text() {
    local file="$1"
    local ext="${file##*.}"
    local relpath="${file#$GDRIVE/}"
    local size=$(stat -f%z "$file" 2>/dev/null || echo 0)

    [[ $size -gt 102400 || $size -eq 0 ]] && return  # Skip >100KB or empty

    local content=""
    case "$ext" in
        json)
            content=$(head -c 2000 "$file" | jq -c '.' 2>/dev/null | head -c 1200)
            [[ -n "$content" ]] && echo "[gdrive:config] $relpath: $content"
            ;;
        sql)
            content=$(grep -E "^(CREATE|ALTER|INSERT|SELECT|--)" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
            [[ ${#content} -gt 30 ]] && echo "[gdrive:schema] $relpath: $content"
            ;;
        py|js|ts)
            content=$(grep -E "^(class |def |function |const |import |from |export )" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
            [[ ${#content} -gt 30 ]] && echo "[gdrive:code:$ext] $relpath: $content"
            ;;
        cpp|c|h|cs|swift)
            content=$(grep -E "^(class |struct |void |int |public |private |#include|#define)" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
            [[ ${#content} -gt 30 ]] && echo "[gdrive:code:$ext] $relpath: $content"
            ;;
        sh)
            content=$(grep -E "^(#|export |function )" "$file" 2>/dev/null | head -10 | tr '\n' ' ')
            [[ ${#content} -gt 20 ]] && echo "[gdrive:script] $relpath: $content"
            ;;
        yaml|yml)
            content=$(head -20 "$file" | tr '\n' ' ' | head -c 800)
            echo "[gdrive:config:yaml] $relpath: $content"
            ;;
        txt|md)
            content=$(head -30 "$file" | tr '\n' ' ' | head -c 1000)
            [[ ${#content} -gt 30 ]] && echo "[gdrive:doc] $relpath: $content"
            ;;
        csv)
            content=$(head -3 "$file" | tr '\n' '|')
            echo "[gdrive:data:csv] $relpath: $content"
            ;;
    esac
}

export -f process_text
export GDRIVE

# Process text files with strict limits per type
log "Processing text files (limited samples)..."

for ext in json py js ts cpp c h cs swift sh yaml yml txt md csv sql; do
    limit=200
    [[ "$ext" == "cpp" || "$ext" == "h" ]] && limit=300
    [[ "$ext" == "json" ]] && limit=400

    log "  Scanning .$ext files (limit $limit)..."
    find "$GDRIVE" -name "*.$ext" -type f -size -100k 2>/dev/null | \
        grep -v -E "(node_modules|__pycache__|\.git|build/|dist/|\.cache)" | \
        head -$limit | \
        while read -r file; do
            process_text "$file"
        done >> "$BATCH_FILE"
done

TEXT_COUNT=$(wc -l < "$BATCH_FILE" | tr -d ' ')
log "Text items extracted: $TEXT_COUNT"

# Catalog binary files (just metadata)
log "Cataloging binary files..."
BINARY_COUNT=0
for ext in zip gz jpg jpeg png gif mp4 mov pdf heic obj dll exe dmg; do
    find "$GDRIVE" -name "*.$ext" -type f 2>/dev/null | \
        grep -v -E "(node_modules|__pycache__|\.git)" | \
        head -100 | \
        while read -r file; do
            relpath="${file#$GDRIVE/}"
            size=$(stat -f%z "$file" 2>/dev/null || echo 0)
            checksum=$(md5 -q "$file" 2>/dev/null | head -c 8 || echo "--------")
            temp=$(mktemp)
            jq --arg p "$relpath" --arg s "$size" --arg e "$ext" --arg c "$checksum" \
               '. + [{path: $p, size: ($s|tonumber), type: $e, checksum: $c}]' \
               "$CATALOG_FILE" > "$temp" && mv "$temp" "$CATALOG_FILE"
            ((BINARY_COUNT++))
        done
done
log "Binary files cataloged: $(jq 'length' "$CATALOG_FILE")"

# Add to MB1
log "Adding to MB1 learned items..."
count=0
while IFS= read -r line; do
    [[ -z "$line" ]] && continue
    escaped=$(echo "$line" | jq -Rs '.')
    timestamp=$(date +%s)
    temp=$(mktemp)
    jq --argjson c "$escaped" --argjson t "$timestamp" \
       '. + [{content: $c, importance: 0.83, timestamp: $t, relevance: 0.5}]' \
       "$MB1_LEARNED" > "$temp" && mv "$temp" "$MB1_LEARNED"
    ((count++))
    [[ $((count % 200)) -eq 0 ]] && log "  $count items processed..."
done < "$BATCH_FILE"

log "Ingested $count items to MB1"

# Compress
log "Compressing..."
zstd -f "$CATALOG_FILE" -o "$BINARY_STORE/catalog.zst" 2>/dev/null
zstd -f "$MB1_LEARNED" -o "$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/thataiguy/learned_items.json.zst" 2>/dev/null

# Send to MB2
log "Sending to MB2..."
head -1000 "$BATCH_FILE" > /tmp/mb2_gdrive.txt
sshpass -p [credential removed] scp -o StrictHostKeyChecking=no /tmp/mb2_gdrive.txt beandaddy@192.168.4.62:/tmp/

sshpass -p [credential removed] ssh -o StrictHostKeyChecking=no -o ConnectTimeout=60 beandaddy@192.168.4.62 << 'REMOTE'
export PATH="/opt/homebrew/bin:$PATH"
LEARNED="$HOME/.seferim/data/learned_items.json"
[[ ! -f "$LEARNED" ]] && echo "[]" > "$LEARNED"
count=0
while IFS= read -r line; do
    [[ -z "$line" ]] && continue
    content=$(echo "$line" | jq -Rs '.')
    timestamp=$(date +%s)
    temp=$(mktemp)
    jq --argjson c "$content" --argjson t "$timestamp" \
       '. + [{content: $c, importance: 0.83, timestamp: $t, relevance: 0.5}]' \
       "$LEARNED" > "$temp" && mv "$temp" "$LEARNED"
    ((count++))
done < /tmp/mb2_gdrive.txt
echo "MB2: $count items ingested"
rm /tmp/mb2_gdrive.txt
REMOTE

rm -f "$BATCH_FILE" /tmp/mb2_gdrive.txt

log "=== INGESTION COMPLETE ==="
log "MB1 total: $(jq 'length' "$MB1_LEARNED") items"