Janady13/AI_Brain / the_tired_agi/core/include/ingest_pipeline.hpp

1// ============================================================================== 2// The-Tired-AGI Core: Ingest Pipeline 3// Read-Only Text Processing & Embedding Generation 4// ============================================================================== 5 6#pragma once 7 8#include <string> 9#include <string_view> 10#include <vector> 11#include <array> 12#include <cstdint> 13#include <optional> 14#include <functional> 15#include <atomic> 16#include <memory> 17#include <filesystem> 18 19namespace tired_agi { 20 21// ============================================================================== 22// Text Chunk 23// ============================================================================== 24 25/// A fixed-size chunk of text with metadata 26struct TextChunk { 27 std::array<uint8_t, 16384> data; // 16 KiB chunk buffer 28 size_t size; // Actual bytes used 29 uint64_t hash; // xxHash64 of content 30 double entropy; // Shannon entropy [0,1] 31 uint64_t chunk_id; // Unique chunk identifier 32 std::string source_file; // Original file path 33 uint64_t file_offset; // Byte offset in file 34 uint64_t timestamp; // Ingestion timestamp 35 36 constexpr static size_t MAX_SIZE = 16384; 37 38 TextChunk() noexcept 39 : size(0), hash(0), entropy(0.0), chunk_id(0), file_offset(0), timestamp(0) { 40 data.fill(0); 41 } 42}; 43 44// ============================================================================== 45// Embedding Vector 46// ============================================================================== 47 48/// Embedding generated from text chunk 49struct Embedding { 50 std::vector<double> vector; // Dense embedding vector 51 uint64_t chunk_id; // Source chunk ID 52 uint64_t hash; // Source chunk hash 53 double magnitude; // L2 norm of vector 54 uint64_t timestamp; // Generation timestamp 55 56 Embedding() noexcept 57 : chunk_id(0), hash(0), magnitude(0.0), timestamp(0) {} 58 59 explicit Embedding(size_t dim) noexcept 60 : vector(dim, 0.0), chunk_id(0), hash(0), magnitude(0.0), timestamp(0) {} 61}; 62 63// ============================================================================== 64// Ingest Statistics 65// ============================================================================== 66 67struct IngestStatistics { 68 uint64_t total_files_scanned; 69 uint64_t total_files_processed; 70 uint64_t total_bytes_read; 71 uint64_t total_chunks_generated; 72 uint64_t total_embeddings_generated; 73 uint64_t total_errors; 74 double average_entropy; 75 double average_chunk_size; 76 double processing_rate_mbps; // Megabytes per second 77 uint64_t elapsed_time_ms; 78 79 IngestStatistics() noexcept 80 : total_files_scanned(0) 81 , total_files_processed(0) 82 , total_bytes_read(0) 83 , total_chunks_generated(0) 84 , total_embeddings_generated(0) 85 , total_errors(0) 86 , average_entropy(0.0) 87 , average_chunk_size(0.0) 88 , processing_rate_mbps(0.0) 89 , elapsed_time_ms(0) {} 90}; 91 92// ============================================================================== 93// Ingest Configuration 94// ============================================================================== 95 96struct IngestConfig { 97 // Chunk processing 98 size_t chunk_size = 16384; // 16 KiB default 99 size_t chunk_overlap = 512; // Overlap between chunks 100 size_t min_chunk_size = 1024; // Minimum chunk size 101 double min_entropy_threshold = 0.1; // Skip low-entropy chunks 102 103 // Embedding generation 104 size_t embedding_dim = 768; // Embedding dimension 105 bool normalize_embeddings = true; // L2 normalize to unit sphere 106 107 // File filtering 108 std::vector<std::string> allowed_extensions = { 109 ".txt", ".md", ".c", ".cpp", ".h", ".hpp", 110 ".go", ".rs", ".py", ".js", ".java", ".cs", 111 ".v", ".sv", ".vhd", ".vhdl" 112 }; 113 size_t max_file_size_mb = 100; // Skip files larger than this 114 bool skip_hidden_files = true; // Skip .hidden files 115 bool skip_binary_files = true; // Detect and skip binary files 116 117 // Performance 118 size_t num_worker_threads = 4; // Parallel processing 119 size_t buffer_size = 1024 * 1024; // 1 MB read buffer 120 bool use_mmap = true; // Memory-mapped file I/O 121 122 // Safety (read-only guarantees) 123 bool read_only = true; // NEVER write to source files 124 bool verify_checksums = true; // Verify data integrity 125}; 126 127// ============================================================================== 128// Ingest Pipeline 129// ============================================================================== 130 131class IngestPipeline { 132public: 133 explicit IngestPipeline(IngestConfig config = {}) noexcept; 134 ~IngestPipeline(); 135 136 // Disable copy, enable move 137 IngestPipeline(const IngestPipeline&) = delete; 138 IngestPipeline& operator=(const IngestPipeline&) = delete; 139 IngestPipeline(IngestPipeline&&) noexcept = default; 140 IngestPipeline& operator=(IngestPipeline&&) noexcept = default; 141 142 // ============================================================================== 143 // File Processing 144 // ============================================================================== 145 146 /// Process a single file: read → chunk → embed 147 std::vector<Embedding> process_file(const std::filesystem::path& path) noexcept; 148 149 /// Process directory recursively 150 std::vector<Embedding> process_directory( 151 const std::filesystem::path& dir, 152 bool recursive = true 153 ) noexcept; 154 155 /// Process multiple files in parallel 156 std::vector<Embedding> process_files_parallel( 157 const std::vector<std::filesystem::path>& paths 158 ) noexcept; 159 160 // ============================================================================== 161 // Low-Level Processing 162 // ============================================================================== 163 164 /// Read file and split into chunks 165 std::vector<TextChunk> read_and_chunk(const std::filesystem::path& path) noexcept; 166 167 /// Generate embedding from text chunk 168 Embedding chunk_to_embedding(const TextChunk& chunk) noexcept; 169 170 /// Batch embedding generation 171 std::vector<Embedding> chunks_to_embeddings(const std::vector<TextChunk>& chunks) noexcept; 172 173 // ============================================================================== 174 // Utilities 175 // ============================================================================== 176 177 /// Compute xxHash64 of data 178 static uint64_t compute_hash(const uint8_t* data, size_t size) noexcept; 179 180 /// Compute Shannon entropy of data 181 static double compute_entropy(const uint8_t* data, size_t size) noexcept; 182 183 /// Detect if file is binary (non-text) 184 static bool is_binary_file(const std::filesystem::path& path) noexcept; 185 186 /// Check if file extension is allowed 187 bool is_allowed_file(const std::filesystem::path& path) const noexcept; 188 189 // ============================================================================== 190 // Statistics and Monitoring 191 // ============================================================================== 192 193 IngestStatistics get_statistics() const noexcept; 194 void reset_statistics() noexcept; 195 196 // ============================================================================== 197 // Configuration 198 // ============================================================================== 199 200 const IngestConfig& config() const noexcept { return config_; } 201 void update_config(const IngestConfig& config) noexcept { config_ = config; } 202 203 // ============================================================================== 204 // Callback Support 205 // ============================================================================== 206 207 using ChunkCallback = std::function<void(const TextChunk&)>; 208 using EmbeddingCallback = std::function<void(const Embedding&)>; 209 using ErrorCallback = std::function<void(const std::string&, const std::string&)>; 210 211 void set_chunk_callback(ChunkCallback callback) noexcept { chunk_callback_ = callback; } 212 void set_embedding_callback(EmbeddingCallback callback) noexcept { embedding_callback_ = callback; } 213 void set_error_callback(ErrorCallback callback) noexcept { error_callback_ = callback; } 214 215private: 216 IngestConfig config_; 217 218 // Callbacks 219 ChunkCallback chunk_callback_; 220 EmbeddingCallback embedding_callback_; 221 ErrorCallback error_callback_; 222 223 // Statistics (atomic for thread safety) 224 mutable std::atomic<uint64_t> stat_files_scanned_{0}; 225 mutable std::atomic<uint64_t> stat_files_processed_{0}; 226 mutable std::atomic<uint64_t> stat_bytes_read_{0}; 227 mutable std::atomic<uint64_t> stat_chunks_generated_{0}; 228 mutable std::atomic<uint64_t> stat_embeddings_generated_{0}; 229 mutable std::atomic<uint64_t> stat_errors_{0}; 230 mutable std::atomic<double> stat_sum_entropy_{0.0}; 231 mutable std::atomic<double> stat_sum_chunk_size_{0.0}; 232 mutable std::atomic<uint64_t> stat_start_time_ms_{0}; 233 234 // Helper functions 235 std::vector<TextChunk> chunk_data( 236 const uint8_t* data, 237 size_t size, 238 const std::string& source_file, 239 uint64_t file_offset 240 ) noexcept; 241 242 Embedding simple_embedding_generation(const TextChunk& chunk) noexcept; 243 void update_statistics(const TextChunk& chunk) noexcept; 244 void update_statistics(const Embedding& embedding) noexcept; 245 void report_error(const std::string& context, const std::string& message) noexcept; 246 247 uint64_t get_time_ms() const noexcept; 248}; 249 250// ============================================================================== 251// xxHash64 Implementation 252// ============================================================================== 253 254/// Fast 64-bit hash function 255class XXHash64 { 256public: 257 explicit XXHash64(uint64_t seed = 0) noexcept; 258 259 void update(const void* data, size_t size) noexcept; 260 uint64_t digest() noexcept; 261 262 static uint64_t hash(const void* data, size_t size, uint64_t seed = 0) noexcept; 263 264private: 265 static constexpr uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL; 266 static constexpr uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL; 267 static constexpr uint64_t PRIME64_3 = 0x165667B19E3779F9ULL; 268 static constexpr uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL; 269 static constexpr uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL; 270 271 uint64_t state_[4]; 272 uint64_t total_len_; 273 uint64_t seed_; 274 uint8_t buffer_[32]; 275 size_t buffer_size_; 276 277 static uint64_t rotl64(uint64_t x, int r) noexcept { 278 return (x << r) | (x >> (64 - r)); 279 } 280 281 static uint64_t round(uint64_t acc, uint64_t input) noexcept { 282 acc += input * PRIME64_2; 283 acc = rotl64(acc, 31); 284 acc *= PRIME64_1; 285 return acc; 286 } 287 288 static uint64_t merge_round(uint64_t acc, uint64_t val) noexcept { 289 val = round(0, val); 290 acc ^= val; 291 acc = acc * PRIME64_1 + PRIME64_4; 292 return acc; 293 } 294 295 static uint64_t avalanche(uint64_t h64) noexcept { 296 h64 ^= h64 >> 33; 297 h64 *= PRIME64_2; 298 h64 ^= h64 >> 29; 299 h64 *= PRIME64_3; 300 h64 ^= h64 >> 32; 301 return h64; 302 } 303}; 304 305} // namespace tired_agi 306