// ==============================================================================
// The-Tired-AGI Core: Ingest Pipeline
// Read-Only Text Processing & Embedding Generation
// ==============================================================================

#pragma once

#include <string>
#include <string_view>
#include <vector>
#include <array>
#include <cstdint>
#include <optional>
#include <functional>
#include <atomic>
#include <memory>
#include <filesystem>

namespace tired_agi {

// ==============================================================================
// Text Chunk
// ==============================================================================

/// A fixed-size chunk of text with metadata
struct TextChunk {
    std::array<uint8_t, 16384> data;    // 16 KiB chunk buffer
    size_t size;                         // Actual bytes used
    uint64_t hash;                       // xxHash64 of content
    double entropy;                      // Shannon entropy [0,1]
    uint64_t chunk_id;                   // Unique chunk identifier
    std::string source_file;             // Original file path
    uint64_t file_offset;                // Byte offset in file
    uint64_t timestamp;                  // Ingestion timestamp

    constexpr static size_t MAX_SIZE = 16384;

    TextChunk() noexcept
        : size(0), hash(0), entropy(0.0), chunk_id(0), file_offset(0), timestamp(0) {
        data.fill(0);
    }
};

// ==============================================================================
// Embedding Vector
// ==============================================================================

/// Embedding generated from text chunk
struct Embedding {
    std::vector<double> vector;          // Dense embedding vector
    uint64_t chunk_id;                   // Source chunk ID
    uint64_t hash;                       // Source chunk hash
    double magnitude;                    // L2 norm of vector
    uint64_t timestamp;                  // Generation timestamp

    Embedding() noexcept
        : chunk_id(0), hash(0), magnitude(0.0), timestamp(0) {}

    explicit Embedding(size_t dim) noexcept
        : vector(dim, 0.0), chunk_id(0), hash(0), magnitude(0.0), timestamp(0) {}
};

// ==============================================================================
// Ingest Statistics
// ==============================================================================

struct IngestStatistics {
    uint64_t total_files_scanned;
    uint64_t total_files_processed;
    uint64_t total_bytes_read;
    uint64_t total_chunks_generated;
    uint64_t total_embeddings_generated;
    uint64_t total_errors;
    double average_entropy;
    double average_chunk_size;
    double processing_rate_mbps;         // Megabytes per second
    uint64_t elapsed_time_ms;

    IngestStatistics() noexcept
        : total_files_scanned(0)
        , total_files_processed(0)
        , total_bytes_read(0)
        , total_chunks_generated(0)
        , total_embeddings_generated(0)
        , total_errors(0)
        , average_entropy(0.0)
        , average_chunk_size(0.0)
        , processing_rate_mbps(0.0)
        , elapsed_time_ms(0) {}
};

// ==============================================================================
// Ingest Configuration
// ==============================================================================

struct IngestConfig {
    // Chunk processing
    size_t chunk_size = 16384;           // 16 KiB default
    size_t chunk_overlap = 512;          // Overlap between chunks
    size_t min_chunk_size = 1024;        // Minimum chunk size
    double min_entropy_threshold = 0.1;  // Skip low-entropy chunks

    // Embedding generation
    size_t embedding_dim = 768;          // Embedding dimension
    bool normalize_embeddings = true;    // L2 normalize to unit sphere

    // File filtering
    std::vector<std::string> allowed_extensions = {
        ".txt", ".md", ".c", ".cpp", ".h", ".hpp",
        ".go", ".rs", ".py", ".js", ".java", ".cs",
        ".v", ".sv", ".vhd", ".vhdl"
    };
    size_t max_file_size_mb = 100;       // Skip files larger than this
    bool skip_hidden_files = true;       // Skip .hidden files
    bool skip_binary_files = true;       // Detect and skip binary files

    // Performance
    size_t num_worker_threads = 4;       // Parallel processing
    size_t buffer_size = 1024 * 1024;    // 1 MB read buffer
    bool use_mmap = true;                // Memory-mapped file I/O

    // Safety (read-only guarantees)
    bool read_only = true;               // NEVER write to source files
    bool verify_checksums = true;        // Verify data integrity
};

// ==============================================================================
// Ingest Pipeline
// ==============================================================================

class IngestPipeline {
public:
    explicit IngestPipeline(IngestConfig config = {}) noexcept;
    ~IngestPipeline();

    // Disable copy, enable move
    IngestPipeline(const IngestPipeline&) = delete;
    IngestPipeline& operator=(const IngestPipeline&) = delete;
    IngestPipeline(IngestPipeline&&) noexcept = default;
    IngestPipeline& operator=(IngestPipeline&&) noexcept = default;

    // ==============================================================================
    // File Processing
    // ==============================================================================

    /// Process a single file: read → chunk → embed
    std::vector<Embedding> process_file(const std::filesystem::path& path) noexcept;

    /// Process directory recursively
    std::vector<Embedding> process_directory(
        const std::filesystem::path& dir,
        bool recursive = true
    ) noexcept;

    /// Process multiple files in parallel
    std::vector<Embedding> process_files_parallel(
        const std::vector<std::filesystem::path>& paths
    ) noexcept;

    // ==============================================================================
    // Low-Level Processing
    // ==============================================================================

    /// Read file and split into chunks
    std::vector<TextChunk> read_and_chunk(const std::filesystem::path& path) noexcept;

    /// Generate embedding from text chunk
    Embedding chunk_to_embedding(const TextChunk& chunk) noexcept;

    /// Batch embedding generation
    std::vector<Embedding> chunks_to_embeddings(const std::vector<TextChunk>& chunks) noexcept;

    // ==============================================================================
    // Utilities
    // ==============================================================================

    /// Compute xxHash64 of data
    static uint64_t compute_hash(const uint8_t* data, size_t size) noexcept;

    /// Compute Shannon entropy of data
    static double compute_entropy(const uint8_t* data, size_t size) noexcept;

    /// Detect if file is binary (non-text)
    static bool is_binary_file(const std::filesystem::path& path) noexcept;

    /// Check if file extension is allowed
    bool is_allowed_file(const std::filesystem::path& path) const noexcept;

    // ==============================================================================
    // Statistics and Monitoring
    // ==============================================================================

    IngestStatistics get_statistics() const noexcept;
    void reset_statistics() noexcept;

    // ==============================================================================
    // Configuration
    // ==============================================================================

    const IngestConfig& config() const noexcept { return config_; }
    void update_config(const IngestConfig& config) noexcept { config_ = config; }

    // ==============================================================================
    // Callback Support
    // ==============================================================================

    using ChunkCallback = std::function<void(const TextChunk&)>;
    using EmbeddingCallback = std::function<void(const Embedding&)>;
    using ErrorCallback = std::function<void(const std::string&, const std::string&)>;

    void set_chunk_callback(ChunkCallback callback) noexcept { chunk_callback_ = callback; }
    void set_embedding_callback(EmbeddingCallback callback) noexcept { embedding_callback_ = callback; }
    void set_error_callback(ErrorCallback callback) noexcept { error_callback_ = callback; }

private:
    IngestConfig config_;

    // Callbacks
    ChunkCallback chunk_callback_;
    EmbeddingCallback embedding_callback_;
    ErrorCallback error_callback_;

    // Statistics (atomic for thread safety)
    mutable std::atomic<uint64_t> stat_files_scanned_{0};
    mutable std::atomic<uint64_t> stat_files_processed_{0};
    mutable std::atomic<uint64_t> stat_bytes_read_{0};
    mutable std::atomic<uint64_t> stat_chunks_generated_{0};
    mutable std::atomic<uint64_t> stat_embeddings_generated_{0};
    mutable std::atomic<uint64_t> stat_errors_{0};
    mutable std::atomic<double> stat_sum_entropy_{0.0};
    mutable std::atomic<double> stat_sum_chunk_size_{0.0};
    mutable std::atomic<uint64_t> stat_start_time_ms_{0};

    // Helper functions
    std::vector<TextChunk> chunk_data(
        const uint8_t* data,
        size_t size,
        const std::string& source_file,
        uint64_t file_offset
    ) noexcept;

    Embedding simple_embedding_generation(const TextChunk& chunk) noexcept;
    void update_statistics(const TextChunk& chunk) noexcept;
    void update_statistics(const Embedding& embedding) noexcept;
    void report_error(const std::string& context, const std::string& message) noexcept;

    uint64_t get_time_ms() const noexcept;
};

// ==============================================================================
// xxHash64 Implementation
// ==============================================================================

/// Fast 64-bit hash function
class XXHash64 {
public:
    explicit XXHash64(uint64_t seed = 0) noexcept;

    void update(const void* data, size_t size) noexcept;
    uint64_t digest() noexcept;

    static uint64_t hash(const void* data, size_t size, uint64_t seed = 0) noexcept;

private:
    static constexpr uint64_t PRIME64_1 = 0x9E3779B185EBCA87ULL;
    static constexpr uint64_t PRIME64_2 = 0xC2B2AE3D27D4EB4FULL;
    static constexpr uint64_t PRIME64_3 = 0x165667B19E3779F9ULL;
    static constexpr uint64_t PRIME64_4 = 0x85EBCA77C2B2AE63ULL;
    static constexpr uint64_t PRIME64_5 = 0x27D4EB2F165667C5ULL;

    uint64_t state_[4];
    uint64_t total_len_;
    uint64_t seed_;
    uint8_t buffer_[32];
    size_t buffer_size_;

    static uint64_t rotl64(uint64_t x, int r) noexcept {
        return (x << r) | (x >> (64 - r));
    }

    static uint64_t round(uint64_t acc, uint64_t input) noexcept {
        acc += input * PRIME64_2;
        acc = rotl64(acc, 31);
        acc *= PRIME64_1;
        return acc;
    }

    static uint64_t merge_round(uint64_t acc, uint64_t val) noexcept {
        val = round(0, val);
        acc ^= val;
        acc = acc * PRIME64_1 + PRIME64_4;
        return acc;
    }

    static uint64_t avalanche(uint64_t h64) noexcept {
        h64 ^= h64 >> 33;
        h64 *= PRIME64_2;
        h64 ^= h64 >> 29;
        h64 *= PRIME64_3;
        h64 ^= h64 >> 32;
        return h64;
    }
};

} // namespace tired_agi