Janady13/complete-agi-core / orchestrator/ingest-downloads.sh

Source identity

Field	Recorded value
Source type	GitHub source
Original record	Janady13/complete-agi-core / orchestrator/ingest-downloads.sh
Date	Not specified in source
Repository	Janady13/complete-agi-core
Revision	33aa84f2aa61cedbe010240ada7f4efbdfe795a7
Branch	claude/improve-math-logic-ehIW1
Path or archive member	orchestrator/ingest-downloads.sh
Original SHA-256	57043efdd9f01da2637173c54a0a96bdb7ef09e0e26656ce8efafddbfdff3bec
Git blob SHA	cb2c2233f9b9697e5303e8983561fed290b5d940
Reading-copy SHA-256	0f9f67a0e244ab5ab8d2a28bb25a38365831a261427018ba6582dae56caffbca

Original source ↗

Recovered source transcript

Download complete source text ↓199 lines · 7.0 KB

Credential-bearing and unrelated personal details were removed from this reading copy. The source identity and original checksum remain recorded.

#!/bin/bash
# DOWNLOADS INGESTION - Extract knowledge, store as binary, empty folder
export PATH="/opt/homebrew/bin:$PATH"

DOWNLOADS="$HOME/Downloads"
BINARY_STORE="$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/downloads_archive"
MB1_LEARNED="$HOME/.agi_core/learned_items.json"
LOG="$HOME/.agi_core/logs/downloads-ingest.log"

mkdir -p "$BINARY_STORE" "$HOME/.agi_core/logs"

log() { echo "[$(date +%H:%M:%S)] $1" | tee -a "$LOG"; }

# Add to learned items
add_learned() {
    local content="$1"
    local importance="${2:-0.80}"
    local timestamp=$(date +%s)
    local escaped=$(echo "$content" | jq -Rs '.')
    local temp=$(mktemp)
    jq --argjson c "$escaped" --argjson i "$importance" --argjson t "$timestamp" \
       '. + [{content: $c, importance: $i, timestamp: $t, relevance: 0.5}]' \
       "$MB1_LEARNED" > "$temp" && mv "$temp" "$MB1_LEARNED"
}

log "=== DOWNLOADS INGESTION STARTING ==="
log "Source: $DOWNLOADS ($(du -sh "$DOWNLOADS" | cut -f1))"

TEXT_COUNT=0
BINARY_COUNT=0
TOTAL_SIZE=0

# Create batch file for extracted content
BATCH_FILE=$(mktemp)
CATALOG_FILE="$BINARY_STORE/catalog_$(date +%Y%m%d_%H%M%S).json"
echo "[]" > "$CATALOG_FILE"

# Process text-based files
process_text() {
    local file="$1"
    local ext="$2"
    local relpath="${file#$DOWNLOADS/}"
    local size=$(stat -f%z "$file" 2>/dev/null || echo 0)

    # Skip files > 500KB for text extraction
    [[ $size -gt 512000 ]] && return 1
    [[ $size -eq 0 ]] && return 1

    local content=""
    case "$ext" in
        json)
            content=$(cat "$file" | jq -c '.' 2>/dev/null | head -c 1500)
            [[ -z "$content" || "$content" == "null" ]] && return 1
            content="[downloads:config] $relpath: $content"
            ;;
        sql)
            content=$(grep -E "^(CREATE|ALTER|INSERT|SELECT|--)" "$file" 2>/dev/null | head -25 | tr '\n' ' ')
            [[ ${#content} -lt 30 ]] && return 1
            content="[downloads:schema] $relpath: $content"
            ;;
        swift|py|cpp|cs|c|js|ts)
            content=$(grep -E "^(class |func |def |public |private |import |using |struct |//|#)" "$file" 2>/dev/null | head -25 | tr '\n' ' ')
            [[ ${#content} -lt 30 ]] && return 1
            content="[downloads:code:$ext] $relpath: $content"
            ;;
        sh|bash)
            content=$(grep -E "^(#|export |function |[A-Z_]+=)" "$file" 2>/dev/null | head -15 | tr '\n' ' ')
            [[ ${#content} -lt 20 ]] && return 1
            content="[downloads:script] $relpath: $content"
            ;;
        yaml|yml)
            content=$(head -30 "$file" | tr '\n' ' ')
            content="[downloads:config:yaml] $relpath: $content"
            ;;
        txt|md)
            content=$(head -40 "$file" | tr '\n' ' ' | head -c 1500)
            [[ ${#content} -lt 20 ]] && return 1
            content="[downloads:doc] $relpath: $content"
            ;;
        csv)
            # Get header + sample rows
            content=$(head -5 "$file" | tr '\n' '|')
            content="[downloads:data:csv] $relpath: $content"
            ;;
        log)
            # Extract key log patterns
            content=$(grep -E "(ERROR|WARN|INFO|START|COMPLETE|SUCCESS|FAIL)" "$file" 2>/dev/null | tail -10 | tr '\n' ' ')
            [[ ${#content} -lt 30 ]] && return 1
            content="[downloads:log] $relpath: $content"
            ;;
        applescript)
            content=$(head -20 "$file" | tr '\n' ' ')
            content="[downloads:script:applescript] $relpath: $content"
            ;;
        *)
            return 1
            ;;
    esac

    if [[ -n "$content" && ${#content} -gt 40 ]]; then
        echo "$content" >> "$BATCH_FILE"
        return 0
    fi
    return 1
}

# Catalog binary/large files
catalog_binary() {
    local file="$1"
    local relpath="${file#$DOWNLOADS/}"
    local size=$(stat -f%z "$file" 2>/dev/null || echo 0)
    local ext="${file##*.}"
    local checksum=$(md5 -q "$file" 2>/dev/null | head -c 8)

    # Add to catalog JSON
    local temp=$(mktemp)
    jq --arg p "$relpath" --arg s "$size" --arg e "$ext" --arg c "$checksum" \
       '. + [{path: $p, size: ($s|tonumber), type: $e, checksum: $c}]' \
       "$CATALOG_FILE" > "$temp" && mv "$temp" "$CATALOG_FILE"
}

log "Processing text files..."
for ext in json sql swift py cpp cs c js ts sh yaml yml txt md csv log applescript; do
    count=0
    while IFS= read -r -d '' file; do
        if process_text "$file" "$ext"; then
            ((count++))
            ((TEXT_COUNT++))
        fi
    done < <(find "$DOWNLOADS" -name "*.$ext" -type f -print0 2>/dev/null)
    [[ $count -gt 0 ]] && log "  $ext: $count files"
done

log "Cataloging binary files..."
for ext in zip jpg jpeg png gif mp4 mov avi dll exe dmg pkg pdf pdb dylib o pyc cache; do
    count=0
    while IFS= read -r -d '' file; do
        catalog_binary "$file"
        ((count++))
        ((BINARY_COUNT++))
    done < <(find "$DOWNLOADS" -name "*.$ext" -type f -print0 2>/dev/null)
    [[ $count -gt 0 ]] && log "  $ext: $count files cataloged"
done

log "Text items extracted: $TEXT_COUNT"
log "Binary files cataloged: $BINARY_COUNT"

# Add all text content to MB1
log "Adding to MB1 learned items..."
while IFS= read -r line; do
    [[ -z "$line" ]] && continue
    add_learned "$line" "0.82"
done < "$BATCH_FILE"

# Compress catalog
log "Compressing catalog..."
zstd -f "$CATALOG_FILE" -o "$BINARY_STORE/catalog.zst" 2>/dev/null

# Add catalog summary to learned items
CATALOG_SUMMARY=$(jq -r 'group_by(.type) | map({type: .[0].type, count: length, total_size: (map(.size) | add)}) | .[]| "\(.type): \(.count) files (\(.total_size/1048576 | floor)MB)"' "$CATALOG_FILE" | tr '\n' ', ')
add_learned "[downloads:archive:catalog] Binary files archived: $CATALOG_SUMMARY" "0.85"

# Compress updated learned_items for iCloud
log "Compressing learned_items for iCloud..."
zstd -f "$MB1_LEARNED" -o "$HOME/Library/Mobile Documents/com~apple~CloudDocs/AGI_Data/agi_core/learned_items.json.zst" 2>/dev/null

# Send to MB2
log "Sending to MB2..."
MB2_BATCH=$(head -2000 "$BATCH_FILE")  # Limit for SSH transfer
sshpass -p [credential removed] ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 beandaddy@192.168.4.62 << REMOTE
export PATH="/opt/homebrew/bin:$PATH"
LEARNED="\$HOME/.seferim/data/learned_items.json"
mkdir -p "\$HOME/.seferim/data"
[[ ! -f "\$LEARNED" ]] && echo "[]" > "\$LEARNED"

while IFS= read -r line; do
    [[ -z "\$line" ]] && continue
    content=\$(echo "\$line" | jq -Rs '.')
    timestamp=\$(date +%s)
    temp=\$(mktemp)
    jq --argjson c "\$content" --argjson t "\$timestamp" \
       '. + [{content: \$c, importance: 0.82, timestamp: \$t, relevance: 0.5}]' \
       "\$LEARNED" > "\$temp" && mv "\$temp" "\$LEARNED"
done << 'BATCHDATA'
$MB2_BATCH
BATCHDATA
echo "MB2 ingestion complete"
REMOTE

rm -f "$BATCH_FILE"

log "=== INGESTION COMPLETE ==="
log "Text content: $TEXT_COUNT items ingested"
log "Binary catalog: $BINARY_COUNT files archived"
log "Catalog stored: $BINARY_STORE/catalog.zst"

# Final counts
log "MB1 total items: $(jq 'length' "$MB1_LEARNED")"