#include #include c_err_t c_NlpLanguageModel_Init(c_NlpLanguageModel_t* self, c_size_t max_n, c_size_t bucket_count) { if (!self || max_n < 2 || bucket_count == 0) return C_ERR_PARAM; self->max_n = max_n; self->backoff_alpha = 0.4; // Industrial standard alpha decay baseline configuration // Allocate space for all N-gram tiers down to 2-gram (size = max_n - 1) c_size_t graph_count = max_n - 1; self->levels = (c_NlpNgramGraph_t*)C_ALLOC(sizeof(c_NlpNgramGraph_t) * graph_count); if (!self->levels) return C_ERR_NOMEM; // Initialize sub-graphs from highest order down to lowest (Bigram) for (c_size_t i = 0; i < graph_count; i++) { c_size_t current_n = max_n - i; c_err_t err = c_NlpNgramGraph_Init(&self->levels[i], current_n, bucket_count, 16); if (err != C_ERR_OK) { // Rollback previously initialized sub-graphs on allocation failure for (c_size_t k = 0; k < i; k++) { c_NlpNgramGraph_Destroy(&self->levels[k]); } C_FREE(self->levels); self->levels = NULL; return err; } } return C_ERR_OK; } c_err_t c_NlpLanguageModel_Train(c_NlpLanguageModel_t* self, const c_ucs4_t* unicode_array, c_size_t length) { if (!self || !self->levels || !unicode_array || length == 0) return C_ERR_PARAM; c_size_t graph_count = self->max_n - 1; // Feed data to train all nested sub-graphs simultaneously for (c_size_t i = 0; i < graph_count; i++) { c_err_t err = c_NlpNgramGraph_Process(&self->levels[i], unicode_array, length); if (err != C_ERR_OK) return err; } return C_ERR_OK; } c_err_t c_NlpLanguageModel_PredictSmooth(c_NlpLanguageModel_t* self, const c_ucs4_t* history_context, c_size_t history_len, c_ucs4_t* out_next_char, double* out_confidence) { if (!self || !self->levels || !history_context || history_len == 0 || !out_next_char || !out_confidence) { return C_ERR_PARAM; } c_size_t graph_count = self->max_n - 1; double current_alpha_multiplier = 1.0; // Step down from the highest-order N-gram down to the baseline Bigram level for (c_size_t i = 0; i < graph_count; i++) { c_NlpNgramGraph_t* current_graph = &self->levels[i]; c_size_t expected_history_len = current_graph->n_value - 1; // If our current available history sequence matches the tier's required context length if (history_len >= expected_history_len) { // Extract a sliced pointer matching exactly the trailing edge suffix of our history c_size_t offset_start = history_len - expected_history_len; const c_ucs4_t* sliced_history = &history_context[offset_start]; double baseline_confidence = 0.0; // Attempt a retrieval match execution path c_err_t err = c_NlpNgramGraph_PredictNext(current_graph, sliced_history, out_next_char, &baseline_confidence); if (err == C_ERR_OK) { // High-order match successful! Deduct confidence with our decay penalty multipliers *out_confidence = baseline_confidence * current_alpha_multiplier; return C_ERR_OK; } } // If the query falls through (C_ERR_FAIL), cascade lower and apply the alpha smoothing penalty current_alpha_multiplier *= self->backoff_alpha; } return C_ERR_FAIL; // Completely out of vocabulary context indicators across all internal graphs } void c_NlpLanguageModel_Destroy(c_NlpLanguageModel_t* self) { if (!self || !self->levels) return; c_size_t graph_count = self->max_n - 1; for (c_size_t i = 0; i < graph_count; i++) { c_NlpNgramGraph_Destroy(&self->levels[i]); } C_FREE(self->levels); self->levels = NULL; self->max_n = 0; }