cAI/cNLP/Library/c_NlpLanguageModel.c

#include <c_NlpLanguageModel.h>
#include <c_Memory.h>


c_err_t c_NlpLanguageModel_Init(c_NlpLanguageModel_t* self, c_size_t max_n, c_size_t bucket_count) {
    if (!self || max_n < 2 || bucket_count == 0) return C_ERR_PARAM;

    self->max_n = max_n;
    self->backoff_alpha = 0.4; // Industrial standard alpha decay baseline configuration

    // Allocate space for all N-gram tiers down to 2-gram (size = max_n - 1)
    c_size_t graph_count = max_n - 1;
    self->levels = (c_NlpNgramGraph_t*)C_ALLOC(sizeof(c_NlpNgramGraph_t) * graph_count);
    if (!self->levels) return C_ERR_NOMEM;

    // Initialize sub-graphs from highest order down to lowest (Bigram)
    for (c_size_t i = 0; i < graph_count; i++) {
        c_size_t current_n = max_n - i;
        c_err_t err = c_NlpNgramGraph_Init(&self->levels[i], current_n, bucket_count, 16);
        if (err != C_ERR_OK) {
            // Rollback previously initialized sub-graphs on allocation failure
            for (c_size_t k = 0; k < i; k++) {
                c_NlpNgramGraph_Destroy(&self->levels[k]);
            }
            C_FREE(self->levels);
            self->levels = NULL;
            return err;
        }
    }
    return C_ERR_OK;
}

c_err_t c_NlpLanguageModel_Train(c_NlpLanguageModel_t* self, const c_ucs4_t* unicode_array, c_size_t length) {
    if (!self || !self->levels || !unicode_array || length == 0) return C_ERR_PARAM;

    c_size_t graph_count = self->max_n - 1;
    // Feed data to train all nested sub-graphs simultaneously
    for (c_size_t i = 0; i < graph_count; i++) {
        c_err_t err = c_NlpNgramGraph_Process(&self->levels[i], unicode_array, length);
        if (err != C_ERR_OK) return err;
    }
    return C_ERR_OK;
}

c_err_t c_NlpLanguageModel_PredictSmooth(c_NlpLanguageModel_t* self, const c_ucs4_t* history_context, c_size_t history_len,
                                         c_ucs4_t* out_next_char, double* out_confidence) {
    if (!self || !self->levels || !history_context || history_len == 0 || !out_next_char || !out_confidence) {
        return C_ERR_PARAM;
    }

    c_size_t graph_count = self->max_n - 1;
    double current_alpha_multiplier = 1.0;

    // Step down from the highest-order N-gram down to the baseline Bigram level
    for (c_size_t i = 0; i < graph_count; i++) {
        c_NlpNgramGraph_t* current_graph = &self->levels[i];
        c_size_t expected_history_len = current_graph->n_value - 1;

        // If our current available history sequence matches the tier's required context length
        if (history_len >= expected_history_len) {
            // Extract a sliced pointer matching exactly the trailing edge suffix of our history
            c_size_t offset_start = history_len - expected_history_len;
            const c_ucs4_t* sliced_history = &history_context[offset_start];

            double baseline_confidence = 0.0;
            // Attempt a retrieval match execution path
            c_err_t err = c_NlpNgramGraph_PredictNext(current_graph, sliced_history, out_next_char, &baseline_confidence);

            if (err == C_ERR_OK) {
                // High-order match successful! Deduct confidence with our decay penalty multipliers
                *out_confidence = baseline_confidence * current_alpha_multiplier;
                return C_ERR_OK;
            }
        }

        // If the query falls through (C_ERR_FAIL), cascade lower and apply the alpha smoothing penalty
        current_alpha_multiplier *= self->backoff_alpha;
    }

    return C_ERR_FAIL; // Completely out of vocabulary context indicators across all internal graphs
}

void c_NlpLanguageModel_Destroy(c_NlpLanguageModel_t* self) {
    if (!self || !self->levels) return;

    c_size_t graph_count = self->max_n - 1;
    for (c_size_t i = 0; i < graph_count; i++) {
        c_NlpNgramGraph_Destroy(&self->levels[i]);
    }
    C_FREE(self->levels);
    self->levels = NULL;
    self->max_n = 0;
}