diff --git a/cKit/Foundation/c_utf8.h b/cKit/Foundation/c_utf8.h index 87983ab..cdf45cc 100644 --- a/cKit/Foundation/c_utf8.h +++ b/cKit/Foundation/c_utf8.h @@ -5,6 +5,14 @@ #include #endif /*INCLUDED_C_BASE_H*/ +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +#ifndef C_UNICODE_MAX +#define C_UNICODE_MAX 0x10FFFFU +#endif + + /* ------------------------------------------------------------------------------------------------------------------ */ /* */ @@ -15,6 +23,14 @@ typedef uint16_t c_uint16_t; /* ------------------------------------------------------------------------------------------------------------------ */ /* */ +C_STATIC_FORCE_INLINE +bool c_is_valid_unicode(uint32_t code) { + // Unicode 码点不能超过 0x10FFFF，且必须排除 UTF-16 代理对范围 (0xD800 ~ 0xDFFF) + if (code > C_UNICODE_MAX) return false; + if (code >= 0xD800 && code <= 0xDFFF) return false; + return true; +} + /** * @brief 获取一个 UTF-8 字符在当前指针位置所占用的实际字节数 (1 ~ 4 字节) */ diff --git a/cNLP/Library/c_NlpCharGraph.c b/cNLP/Library/c_NlpCharGraph.c new file mode 100644 index 0000000..54472d6 --- /dev/null +++ b/cNLP/Library/c_NlpCharGraph.c @@ -0,0 +1,704 @@ +#include +#include +#include +#include "c_StringBuffer.h" +#include "c_utf8_file.h" + + +C_STATIC_FORCE_INLINE +c_size_t c_NlpGraph_Hash(c_ucs4_t cp, c_size_t capacity) { + return (c_size_t)(cp % capacity); +} + +C_STATIC_FORCE_INLINE +c_err_t c_NlpOffsetList_Init(c_NlpOffsetList_t* list) { + list->count = 0; + list->capacity = 4; + list->offsets = (c_size_t*)C_ALLOC(sizeof(c_size_t) * list->capacity); + return list->offsets ? C_ERR_OK : C_ERR_NOMEM; +} + +static c_err_t c_NlpOffsetList_Add(c_NlpOffsetList_t* list, c_size_t offset) { + if (list->count >= list->capacity) { + c_size_t new_cap = list->capacity * 2; + c_size_t* new_offsets = (c_size_t*)C_ALLOC(sizeof(c_size_t) * new_cap); + if (!new_offsets) return C_ERR_NOMEM; + memcpy(new_offsets, list->offsets, sizeof(c_size_t) * list->count); + C_FREE(list->offsets); + list->offsets = new_offsets; + list->capacity = new_cap; + } + list->offsets[list->count++] = offset; + return C_ERR_OK; +} + +/* --- Dynamic Node Instantiation Engine --- */ +static c_NlpCharNode_t* c_NlpCharNode_Create(c_ucs4_t cp, c_size_t relation_capacity) { + if (relation_capacity == 0) return NULL; + + c_NlpCharNode_t* node = (c_NlpCharNode_t*)C_ALLOC(sizeof(c_NlpCharNode_t)); + if (!node) return NULL; + + node->codepoint = cp; + node->frequency = 0; + node->relation_capacity = relation_capacity; + + if (c_NlpOffsetList_Init(&node->offset_list) != C_ERR_OK) { + C_FREE(node); + return NULL; + } + + // Allocate variable-width hash array tables on the heap + node->next_chars = (c_NlpCharEdge_t**)C_ALLOC(sizeof(c_NlpCharEdge_t*) * relation_capacity); + node->prev_chars = (c_NlpCharEdge_t**)C_ALLOC(sizeof(c_NlpCharEdge_t*) * relation_capacity); + + if (!node->next_chars || !node->prev_chars) { + C_FREE(node->next_chars); + C_FREE(node->prev_chars); + C_FREE(node->offset_list.offsets); + C_FREE(node); + return NULL; + } + + // Zero out pointers across all newly configured custom slots + for (c_size_t i = 0; i < relation_capacity; i++) { + node->next_chars[i] = NULL; + node->prev_chars[i] = NULL; + } + + return node; +} + +static c_NlpCharNode_t* c_NlpCharGraph_GetOrCreateNode(c_NlpCharGraph_t* self, c_ucs4_t cp) { + c_size_t idx = c_NlpGraph_Hash(cp, self->bucket_count); + c_NlpCharNode_t* curr = self->buckets[idx]; + c_NlpCharNode_t* prev = NULL; + + // 沿拉链向下查找匹配的码点 + while (curr != NULL) { + if (curr->codepoint == cp) { + return curr; // 完美命中已有节点 + } + prev = curr; + curr = curr->next; + } + + // 未命中冲突，创建全新节点并通过头插法/尾插法挂入拉链 + c_NlpCharNode_t* new_node = c_NlpCharNode_Create(cp, self->relation_default_capacity); + if (!new_node) return NULL; + new_node->next = NULL; + + if (prev == NULL) { + self->buckets[idx] = new_node; // 桶内第一个节点 + } else { + prev->next = new_node; // 挂在冲突链条末尾 + } + + self->total_unique++; + return new_node; +} + +static c_err_t c_NlpCharNode_AddRelation(c_NlpCharEdge_t** edge_buckets, c_size_t capacity, c_ucs4_t target_cp) { + c_size_t idx = c_NlpGraph_Hash(target_cp, capacity); + c_NlpCharEdge_t* curr = edge_buckets[idx]; + + while (curr != NULL) { + if (curr->target_cp == target_cp) { + curr->co_count++; + return C_ERR_OK; + } + curr = curr->next; + } + + c_NlpCharEdge_t* new_edge = (c_NlpCharEdge_t*)C_ALLOC(sizeof(c_NlpCharEdge_t)); + if (!new_edge) return C_ERR_NOMEM; + + new_edge->target_cp = target_cp; + new_edge->co_count = 1; + new_edge->next = edge_buckets[idx]; + edge_buckets[idx] = new_edge; + return C_ERR_OK; +} + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +c_err_t c_NlpCharGraph_Init(c_NlpCharGraph_t* self, c_size_t bucket_count, c_size_t relation_default_capacity) { + if (!self || bucket_count == 0 || relation_default_capacity == 0) return C_ERR_PARAM; + + self->bucket_count = bucket_count; + self->relation_default_capacity = relation_default_capacity; + self->total_unique = 0; + self->total_words = 0; + + self->buckets = (c_NlpCharNode_t**)C_ALLOC(sizeof(c_NlpCharNode_t*) * bucket_count); + if (!self->buckets) return C_ERR_NOMEM; + + for (c_size_t i = 0; i < bucket_count; i++) { + self->buckets[i] = NULL; + } + return C_ERR_OK; +} + +c_NlpCharNode_t* c_NlpCharGraph_GetNode(c_NlpCharGraph_t* self, c_ucs4_t cp) { + if (!self || !self->buckets) return NULL; + c_size_t idx = c_NlpGraph_Hash(cp, self->bucket_count); + c_NlpCharNode_t* curr = self->buckets[idx]; + while (curr != NULL) { + if (curr->codepoint == cp) return curr; + curr = curr->next; + } + return NULL; +} + +c_err_t c_NlpCharGraph_Process(c_NlpCharGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length) { + if (!self || !self->buckets || !unicode_array || length == 0) return C_ERR_PARAM; + + self->total_words += length; + + for (c_size_t i = 0; i < length; i++) { + c_ucs4_t curr_cp = unicode_array[i]; + + // 1. 通过拉链机制获取/创建当前节点 + c_NlpCharNode_t* curr_node = c_NlpCharGraph_GetOrCreateNode(self, curr_cp); + if (!curr_node) return C_ERR_NOMEM; + + curr_node->frequency++; + c_err_t err = c_NlpOffsetList_Add(&curr_node->offset_list, i); + if (err != C_ERR_OK) return err; + + // 2. 建立前序边，同时保证前序字在拉链表中不丢失 + if (i > 0) { + c_ucs4_t prev_cp = unicode_array[i - 1]; + c_NlpCharNode_t* p_node = c_NlpCharGraph_GetOrCreateNode(self, prev_cp); + if (!p_node) return C_ERR_NOMEM; + + err = c_NlpCharNode_AddRelation(curr_node->prev_chars, curr_node->relation_capacity, prev_cp); + if (err != C_ERR_OK) return err; + } + + // 3. 建立后续边（如“大”关联“模”），同时通过拉链确保“模”字被注册激活 + if (i < length - 1) { + c_ucs4_t next_cp = unicode_array[i + 1]; + c_NlpCharNode_t* n_node = c_NlpCharGraph_GetOrCreateNode(self, next_cp); + if (!n_node) return C_ERR_NOMEM; + + err = c_NlpCharNode_AddRelation(curr_node->next_chars, curr_node->relation_capacity, next_cp); + if (err != C_ERR_OK) return err; + } + } + return C_ERR_OK; +} + + +static void c_NlpCharEdge_FreeBuckets(c_NlpCharEdge_t** edge_buckets, c_size_t capacity) { + if (!edge_buckets) return; + for (c_size_t i = 0; i < capacity; i++) { + c_NlpCharEdge_t* curr = edge_buckets[i]; + while (curr != NULL) { + c_NlpCharEdge_t* temp = curr->next; + C_FREE(curr); + curr = temp; + } + } +} + +void c_NlpCharGraph_Destroy(c_NlpCharGraph_t* self) { + if (!self || !self->buckets || self->bucket_count==0) return; + + for (c_size_t i = 0; i < self->bucket_count; i++) { + c_NlpCharNode_t* node = self->buckets[i]; + + // 沿拉链释放所有冲突节点的深层堆空间 + while (node != NULL) { + c_NlpCharNode_t* next_node = node->next; // 暂存下一个指针 + + C_FREE(node->offset_list.offsets); + c_NlpCharEdge_FreeBuckets(node->next_chars, node->relation_capacity); + c_NlpCharEdge_FreeBuckets(node->prev_chars, node->relation_capacity); + + C_FREE(node->next_chars); + C_FREE(node->prev_chars); + C_FREE(node); + + node = next_node; + } + self->buckets[i] = NULL; + } + C_FREE(self->buckets); + self->bucket_count = 0; + self->total_unique = 0; + self->total_words = 0; +} + + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + + +c_err_t c_NlpCharGraph_AnalyzeDistanceWindow(c_NlpCharGraph_t* self, c_ucs4_t cp1, c_ucs4_t cp2, c_size_t window_size, + c_size_t* out_min_dist, double* out_avg_dist, c_size_t* out_co_occurrences, + c_size_t* out_cp1_precedes_count, c_size_t* out_cp2_precedes_count) { + // 1. Defend against parameter null configurations or invalid window constraints + if (!self || !out_min_dist || !out_avg_dist || !out_co_occurrences || + !out_cp1_precedes_count || !out_cp2_precedes_count || window_size == 0) { + return C_ERR_PARAM; + } + + // Initialize all metrics to zero out out-parameters safely + *out_min_dist = (c_size_t)-1; + *out_avg_dist = 0.0; + *out_co_occurrences = 0; + *out_cp1_precedes_count = 0; + *out_cp2_precedes_count = 0; + + // 2. Fetch the structural node contexts from the master hash bucket allocations + c_NlpCharNode_t* n1 = c_NlpCharGraph_GetNode(self, cp1); + c_NlpCharNode_t* n2 = c_NlpCharGraph_GetNode(self, cp2); + if (!n1 || !n2 || n1->offset_list.count == 0 || n2->offset_list.count == 0) { + return C_ERR_OK; // Return cleanly with zeroed structures if any token is absent + } + + c_size_t* o1 = n1->offset_list.offsets; + c_size_t* o2 = n2->offset_list.offsets; + c_size_t len1 = n1->offset_list.count; + c_size_t len2 = n2->offset_list.count; + + c_size_t min_dist = (c_size_t)-1; + double total_dist_sum = 0.0; + c_size_t window_crossings = 0; + c_size_t cp1_precedes = 0; + c_size_t cp2_precedes = 0; + + // 3. Deploy the Bounded Look-Ahead Sliding Scanner + // Loop through every single occurrence of cp1 + for (c_size_t i = 0; i < len1; i++) { + c_size_t pos1 = o1[i]; + + // Find the first index in o2 where the element might be in range: pos2 >= pos1 - window_size + // We initialize look_idx using a basic optimization strategy, or start from 0 + c_size_t look_idx = 0; + + while (look_idx < len2) { + c_size_t pos2 = o2[look_idx]; + + // Condition A: pos2 is too far behind pos1, keep advancing the scan + if (pos2 < pos1 && (pos1 - pos2) > window_size) { + look_idx++; + continue; + } + + // Condition B: pos2 is too far ahead of pos1, we can stop evaluating for this pos1 + if (pos2 > pos1 && (pos2 - pos1) > window_size) { + break; + } + + // Condition C: pos2 falls within the safe sliding context window boundary limit! + c_size_t dist = (pos1 > pos2) ? (pos1 - pos2) : (pos2 - pos1); + + if (dist < min_dist) { + min_dist = dist; + } + + total_dist_sum += (double)dist; + window_crossings++; + + if (pos1 < pos2) { + cp1_precedes++; // cp1 appears BEFORE cp2 + } else if (pos1 > pos2) { + cp2_precedes++; // cp2 appears BEFORE cp1 + } else { + // Identity overlaps count as neutral crossings (rare for unique terms) + } + + look_idx++; + } + } + + // 4. Populate analytics result payloads accurately + if (window_crossings > 0) { + *out_min_dist = min_dist; + *out_avg_dist = total_dist_sum / (double)window_crossings; + *out_co_occurrences = window_crossings; + *out_cp1_precedes_count = cp1_precedes; + *out_cp2_precedes_count = cp2_precedes; + } + + return C_ERR_OK; +} + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +c_err_t c_NlpCharGraph_ExportJSON(c_NlpCharGraph_t* self, const char* filepath) { + if (!self || !self->buckets || !filepath) return C_ERR_PARAM; + + c_StringBuffer_t sb; + c_err_t err = c_StringBuffer_Init(&sb, 4096); + if (err != C_ERR_OK) return err; + + err = c_StringBuffer_AppendStr(&sb, "{\n"); + err |= c_StringBuffer_AppendStr(&sb, " \"meta\": {\n"); + + char meta_buf[128]; + snprintf(meta_buf, sizeof(meta_buf), " \"total_words\": %lu,\n \"total_unique\": %lu\n", + (unsigned long)self->total_words, (unsigned long)self->total_unique); + err |= c_StringBuffer_AppendStr(&sb, meta_buf); + err |= c_StringBuffer_AppendStr(&sb, " },\n"); + err |= c_StringBuffer_AppendStr(&sb, " \"nodes\": [\n"); + if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; } + + c_size_t nodes_serialized = 0; + + // 外层遍历所有桶 + for (c_size_t i = 0; i < self->bucket_count; i++) { + c_NlpCharNode_t* node = self->buckets[i]; + + // 核心修正：内层沿着冲突拉链链表深度追踪，把每一个被掩埋的字（如“模”）都全部挖出来导出 + while (node != NULL) { + char utf8_word[8]; + c_size_t written_len = 0; + if (c_utf8_from_unicode(node->codepoint, utf8_word, &written_len) != C_ERR_OK) { + node = node->next; + continue; + } + + if (nodes_serialized > 0) { + err |= c_StringBuffer_AppendStr(&sb, ",\n"); + } + + err |= c_StringBuffer_AppendStr(&sb, " {\n"); + + char node_info[256]; + if (utf8_word[0] == '"' || utf8_word[0] == '\\') { + snprintf(node_info, sizeof(node_info), + " \"char\": \"\\%s\",\n \"codepoint\": %u,\n \"frequency\": %lu,\n", + utf8_word, node->codepoint, (unsigned long)node->frequency); + } else { + snprintf(node_info, sizeof(node_info), + " \"char\": \"%s\",\n \"codepoint\": %u,\n \"frequency\": %lu,\n", + utf8_word, node->codepoint, (unsigned long)node->frequency); + } + err |= c_StringBuffer_AppendStr(&sb, node_info); + + // 序列化 offsets + err |= c_StringBuffer_AppendStr(&sb, " \"offsets\": ["); + for (c_size_t k = 0; k < node->offset_list.count; k++) { + char val_buf[32]; + snprintf(val_buf, sizeof(val_buf), "%lu%s", + (unsigned long)node->offset_list.offsets[k], + (k + 1 < node->offset_list.count) ? ", " : ""); + err |= c_StringBuffer_AppendStr(&sb, val_buf); + } + err |= c_StringBuffer_AppendStr(&sb, "],\n"); + + // 序列化 next_chars + err |= c_StringBuffer_AppendStr(&sb, " \"next_chars\": {"); + c_bool_t edge_written = C_FALSE; + + for (c_size_t bucket_idx = 0; bucket_idx < node->relation_capacity; bucket_idx++) { + c_NlpCharEdge_t* edge = node->next_chars[bucket_idx]; + while (edge != NULL) { + char edge_utf8[8]; + c_size_t e_len = 0; + if (c_utf8_from_unicode(edge->target_cp, edge_utf8, &e_len) == C_ERR_OK) { + char edge_buf[128]; + const char* comma_prefix = edge_written ? ", " : ""; + + if (edge_utf8[0] == '"' || edge_utf8[0] == '\\') { + snprintf(edge_buf, sizeof(edge_buf), "%s\"\\%s\": %lu", + comma_prefix, edge_utf8, (unsigned long)edge->co_count); + } else { + snprintf(edge_buf, sizeof(edge_buf), "%s\"%s\": %lu", + comma_prefix, edge_utf8, (unsigned long)edge->co_count); + } + err |= c_StringBuffer_AppendStr(&sb, edge_buf); + edge_written = C_TRUE; + } + edge = edge->next; + } + } + err |= c_StringBuffer_AppendStr(&sb, "}\n"); + err |= c_StringBuffer_AppendStr(&sb, " }"); + + nodes_serialized++; + + // 顺着冲突拉链移动到下一个发生哈希冲突的字节点 + node = node->next; + } + if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; } + } + + err |= c_StringBuffer_AppendStr(&sb, "\n ]\n}\n"); + if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; } + + FILE* file = fopen(filepath, "wb"); + if (!file) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; } + + c_size_t expected_size = sb.size; + size_t written = fwrite(sb.buffer, 1, expected_size, file); + fclose(file); + c_StringBuffer_Destroy(&sb); + + if (written != expected_size) return C_ERR_FAIL; + return C_ERR_OK; +} + + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +/** + * @brief Helper utility to safely extract value payloads trapped between specific JSON keys and brackets. + * Example: transforms ' "char": "大", ' into '大' + */ +static void c_Nlp_ExtractJsonValue(const char* line, const char* key, char* dest_val, size_t dest_max) { + dest_val[0] = '\0'; + char* key_pos = strstr(line, key); + if (!key_pos) return; + + char* start = strchr(key_pos + strlen(key), ':'); + if (!start) return; + start++; // Step past ':' + + // Skip white spaces and opening quotes + while (*start == ' ' || *start == '\t' || *start == '"') { + start++; + } + + size_t idx = 0; + while (*start != '\0' && *start != '"' && *start != ',' && *start != '\n' && *start != '\r' && *start != ']' && idx < dest_max - 1) { + // Handle escaped JSON string properties + if (*start == '\\' && *(start + 1) != '\0') { + start++; + } + dest_val[idx++] = *start++; + } + dest_val[idx] = '\0'; + + // Trim trailing tracking spaces if any present + while (idx > 0 && (dest_val[idx - 1] == ' ' || dest_val[idx - 1] == '\t' || dest_val[idx - 1] == '}')) { + dest_val[--idx] = '\0'; + } +} + + + +c_err_t c_NlpCharGraph_ImportJSON(c_NlpCharGraph_t* self, const char* filepath, c_size_t bucket_count) { + if (!self || !filepath || bucket_count == 0) return C_ERR_PARAM; + + FILE* file = fopen(filepath, "rb"); + if (!file) return C_ERR_FAIL; + + // Detect and safely strip standard UTF-8 BOM if present + unsigned char check_bom[3]; + if (fread(check_bom, 1, 3, file) == 3 && check_bom[0] == 0xEF && check_bom[1] == 0xBB && check_bom[2] == 0xBF) { + // BOM skipped successfully + } else { + fseek(file, 0, SEEK_SET); + } + + c_StringBuffer_t line_sb; + c_err_t err = c_StringBuffer_Init(&line_sb, 512); + if (err != C_ERR_OK) { + fclose(file); + return err; + } + + c_size_t total_words = 0; + c_size_t relation_cap = 16; // Default initialization relation table width + + // Fixed Buffer Upgrade: Secure 4096-byte stack arrays to completely negate truncation risks + char val_buf[4096]; + + // Phase 1: Parse Global Metadata Attributes + while (c_utf8_file_readline(file, &line_sb) == C_ERR_OK) { + if (strstr(line_sb.buffer, "\"total_words\"") != NULL) { + c_Nlp_ExtractJsonValue(line_sb.buffer, "\"total_words\"", val_buf, sizeof(val_buf)); + total_words = (c_size_t)strtoul(val_buf, NULL, 10); + } + if (strstr(line_sb.buffer, "\"nodes\"") != NULL) { + break; // Metadata block parsed, moving onto array entries + } + } + + // Phase 2: Structural Graph Memory Instantiation + err = c_NlpCharGraph_Init(self, bucket_count, relation_cap); + if (err != C_ERR_OK) { + c_StringBuffer_Destroy(&line_sb); + fclose(file); + return err; + } + self->total_words = total_words; + + c_NlpCharNode_t* active_node = NULL; + + // Phase 3: Sequential Structural Node Deserialization + while (c_utf8_file_readline(file, &line_sb) == C_ERR_OK) { + // Ignore standalone array boundaries markers + if (strstr(line_sb.buffer, "{") != NULL && strstr(line_sb.buffer, "\"char\"") == NULL) { + continue; + } + + // Parse individual token parameters + if (strstr(line_sb.buffer, "\"codepoint\"") != NULL) { + c_Nlp_ExtractJsonValue(line_sb.buffer, "\"codepoint\"", val_buf, sizeof(val_buf)); + c_ucs4_t cp = (c_ucs4_t)strtoul(val_buf, NULL, 10); + + // Reconstruct utilizing standard chaining allocator to secure the linked pointers list + active_node = c_NlpCharGraph_GetOrCreateNode(self, cp); + if (!active_node) { + c_NlpCharGraph_Destroy(self); + c_StringBuffer_Destroy(&line_sb); + fclose(file); + return C_ERR_NOMEM; + } + } + else if (strstr(line_sb.buffer, "\"frequency\"") != NULL && active_node) { + c_Nlp_ExtractJsonValue(line_sb.buffer, "\"frequency\"", val_buf, sizeof(val_buf)); + active_node->frequency = (c_size_t)strtoul(val_buf, NULL, 10); + } + else if (strstr(line_sb.buffer, "\"offsets\"") != NULL && active_node) { + c_Nlp_ExtractJsonValue(line_sb.buffer, "\"offsets\"", val_buf, sizeof(val_buf)); + + // Safe continuous array chunk tokenization using reentrant strtok mechanics + char* save_ptr = NULL; + char* token = c_utf8_strtok(val_buf, ", ", &save_ptr); + while (token != NULL) { + if (*token != '\0') { + c_size_t offset_val = (c_size_t)strtoul(token, NULL, 10); + c_NlpOffsetList_Add(&active_node->offset_list, offset_val); + } + token = c_utf8_strtok(NULL, ", ", &save_ptr); + } + } + else if (strstr(line_sb.buffer, "\"next_chars\"") != NULL && active_node) { + c_Nlp_ExtractJsonValue(line_sb.buffer, "\"next_chars\"", val_buf, sizeof(val_buf)); + + // Process directional map entries: e.g., '"I": 1, "型": 1' + char* save_ptr = NULL; + char* pair_token = c_utf8_strtok(val_buf, ", ", &save_ptr); + while (pair_token != NULL) { + char* colon = strchr(pair_token, ':'); + if (colon) { + *colon = '\0'; + char edge_char[4096]; // Secure localized tracking array width + strncpy(edge_char, pair_token, sizeof(edge_char) - 1); + edge_char[sizeof(edge_char) - 1] = '\0'; + + // Strip enclosing quote strings out of target boundaries + char* edge_clean = edge_char; + while (*edge_clean == ' ' || *edge_clean == '\t' || *edge_clean == '"') { + edge_clean++; + } + size_t elen = strlen(edge_clean); + while (elen > 0 && (edge_clean[elen - 1] == '"' || edge_clean[elen - 1] == ' ' || edge_clean[elen - 1] == '\t')) { + edge_clean[--elen] = '\0'; + } + + if (elen > 0) { + c_ucs4_t target_cp = 0; + c_size_t bytes_step = 0; + // Transform character text string representation straight back into standard 32-bit integer points + if (c_utf8_to_unicode(edge_clean, &target_cp, &bytes_step) == C_ERR_OK) { + c_size_t edge_freq = (c_size_t)strtoul(colon + 1, NULL, 10); + + // Rehydrate co-occurrence link matrices + for (c_size_t f = 0; f < edge_freq; f++) { + c_NlpCharNode_AddRelation(active_node->next_chars, active_node->relation_capacity, target_cp); + } + } + } + } + pair_token = c_utf8_strtok(NULL, ", ", &save_ptr); + } + } + } + + c_StringBuffer_Destroy(&line_sb); + fclose(file); + return C_ERR_OK; +} + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +c_err_t c_NlpCharGraph_PredictNext(c_NlpCharGraph_t* self, c_ucs4_t current_char, c_NlpPredictMode_t mode, + c_ucs4_t* out_next_char, double* out_confidence) { + if (!self || !out_next_char || !out_confidence) { + return C_ERR_PARAM; + } + + // 1. 在主哈希拉链表中定位当前字的节点 + c_NlpCharNode_t* node = c_NlpCharGraph_GetNode(self, current_char); + if (!node || node->frequency == 0) { + return C_ERR_FAIL; // 图中不存在这个字，或者该字从未作为主词出现过 + } + + // 2. 遍历该节点的所有 next_chars 边，统计后续字的总权重和最大频次 + c_size_t total_co_count = 0; + c_size_t max_co_count = 0; + c_ucs4_t best_target_cp = 0; + + for (c_size_t i = 0; i < node->relation_capacity; i++) { + c_NlpCharEdge_t* edge = node->next_chars[i]; + while (edge != NULL) { + total_co_count += edge->co_count; + + // 贪婪模式所需的条件：记录最大频次 + if (edge->co_count > max_co_count) { + max_co_count = edge->co_count; + best_target_cp = edge->target_cp; + } + edge = edge->next; + } + } + + // 如果该字没有任何后续关联（比如它是文章的最后一个字且仅出现了一次） + if (total_co_count == 0) { + return C_ERR_FAIL; + } + + // 3. 根据预测模式确定输出 + if (mode == C_PREDICT_MODE_GREEDY) { + // 【贪婪模式】：直接选择后序关联最强、出现次数最多的字 + *out_next_char = best_target_cp; + *out_confidence = (double)max_co_count / (double)total_co_count; + } + else if (mode == C_PREDICT_MODE_SAMPLING) { + // 【随机抽样模式】：基于一阶马尔可夫链的条件概率分布进行轮盘赌抽样 + // 生成一个 0.0 到 1.0 之间的随机数 + double random_pivot = (double)rand() / (double)RAND_MAX; + double current_accumulator = 0.0; + c_bool_t sampled = C_FALSE; + + for (c_size_t i = 0; i < node->relation_capacity; i++) { + c_NlpCharEdge_t* edge = node->next_chars[i]; + while (edge != NULL) { + double probability = (double)edge->co_count / (double)total_co_count; + current_accumulator += probability; + + // 落在当前随机区间内，命中该字符 + if (random_pivot <= current_accumulator) { + *out_next_char = edge->target_cp; + *out_confidence = probability; + sampled = C_TRUE; + break; + } + edge = edge->next; + } + if (sampled) break; + } + + // 兜底保护，防止浮点数精度截断导致未命中 + if (!sampled) { + *out_next_char = best_target_cp; + *out_confidence = (double)max_co_count / (double)total_co_count; + } + } else { + return C_ERR_PARAM; + } + + return C_ERR_OK; +} + diff --git a/cNLP/Library/c_NlpCharGraph.h b/cNLP/Library/c_NlpCharGraph.h new file mode 100644 index 0000000..1e1eed6 --- /dev/null +++ b/cNLP/Library/c_NlpCharGraph.h @@ -0,0 +1,116 @@ +#ifndef INCLUDED_C_NLPCHARGRAPH_H +#define INCLUDED_C_NLPCHARGRAPH_H + + +#ifndef INCLUDED_C_BASE_H +#include +#endif /*INCLUDED_C_BASE_H*/ + +#ifndef INCLUDED_C_UTF8_H +#include +#endif /*INCLUDED_C_UTF8_H*/ + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +typedef enum { + C_PREDICT_MODE_GREEDY = 0, /* 贪婪模式：总是选择频次最高的最优解 */ + C_PREDICT_MODE_SAMPLING = 1 /* 抽样模式：根据概率分布进行随机选择（更具多样性） */ +} c_NlpPredictMode_t; + +/* --- 1. 动态位置数组（记录某个字在文章中出现的所有索引位置） --- */ +typedef struct { + c_size_t* offsets; // 存储绝对位置的数组（如：第0, 15, 234个字） + c_size_t count; // 出现频次 (次数) + c_size_t capacity; // 动态数组容量 +} c_NlpOffsetList_t; + +/* --- 2. 邻接边结构体（记录当前字与前后字的关系及共现次数） --- */ +typedef struct c_NlpCharEdge_t { + c_ucs4_t target_cp; // 关联字的 Unicode 码点 + c_size_t co_count; // 在邻接上下文中共同出现的次数 + struct c_NlpCharEdge_t* next; // 冲突链表指针（使用哈希拉链法或链表） +} c_NlpCharEdge_t; + +/* --- 3. 核心：Unicode 字符语义拓扑节点 --- */ +typedef struct c_NlpCharNode_t{ + c_ucs4_t codepoint; + c_size_t frequency; + c_NlpOffsetList_t offset_list; + + // Refactored pointers managing arbitrarily sized hash bucket tables + c_NlpCharEdge_t** next_chars; // Forward Bigram Hash Buckets array + c_NlpCharEdge_t** prev_chars; // Backward Bigram Hash Buckets array + c_size_t relation_capacity; // Current allocated size for next_chars and prev_chars + + struct c_NlpCharNode_t* next; +} c_NlpCharNode_t; + +typedef struct { + c_NlpCharNode_t** buckets; // 主哈希表：存储文章中所有出现过的唯一字节点 + c_size_t bucket_count; // 主哈希桶大小（如 1024 或 4096，取决于去重后的字数） + c_size_t relation_default_capacity; // The default bucket capacity configured for sub-nodes + c_size_t total_unique; // 文章中不重复的汉字总量 + c_size_t total_words; // 整篇文章的总字数（包括标点） +} c_NlpCharGraph_t; + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +c_err_t c_NlpCharGraph_Init(c_NlpCharGraph_t* self, c_size_t bucket_count, c_size_t relation_default_capacity); +c_NlpCharNode_t* c_NlpCharGraph_GetNode(c_NlpCharGraph_t* self, c_ucs4_t cp); +c_err_t c_NlpCharGraph_Process(c_NlpCharGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length); +void c_NlpCharGraph_Destroy(c_NlpCharGraph_t* self); + +/** + * @brief Analyzes structural proximity metrics between two tokens within a sliding context window. + * @param self Pointer to the active initialized graph context block. + * @param cp1 The reference source Unicode character point. + * @param cp2 The target matching Unicode character point to measure. + * @param window_size Maximum token span distance threshold allowed (e.g., 5 tokens). + * @param out_min_dist Destination tracking pointer for the absolute minimum token distance. + * @param out_avg_dist Destination tracking pointer for the calculated average token distance. + * @param out_co_occurrences Destination tracking pointer counting total window crossing events. + * @param out_cp1_precedes_count Tracks how many times cp1 appeared BEFORE cp2 within the window. + * @param out_cp2_precedes_count Tracks how many times cp2 appeared BEFORE cp1 within the window. + * @return c_err_t C_ERR_OK on successful analytical completion, or C_ERR_PARAM if arguments are invalid. + */ +c_err_t c_NlpCharGraph_AnalyzeDistanceWindow(c_NlpCharGraph_t* self, c_ucs4_t cp1, c_ucs4_t cp2, c_size_t window_size, + c_size_t* out_min_dist, double* out_avg_dist, c_size_t* out_co_occurrences, + c_size_t* out_cp1_precedes_count, c_size_t* out_cp2_precedes_count); + +/** + * @brief Exports the entire Unicode semantics graph structure to a file in valid JSON format. + * Automatically translates 32-bit codepoints back to human-readable UTF-8 strings. + * @param self Pointer to the active processed graph context block. + * @param filepath Destination path on disk where the JSON file will be saved. + * @return c_err_t C_ERR_OK on complete success, C_ERR_PARAM on invalid parameters, or C_ERR_FAIL on file access errors. + */ +c_err_t c_NlpCharGraph_ExportJSON(c_NlpCharGraph_t* self, const char* filepath); + +/** + * @brief Parses an exported JSON text file and reconstructs the full Unicode graph structure in memory. + * @param self Pointer to an uninitialized or empty c_NlpCharGraph_t instance. + * @param filepath Path to the source JSON file on disk. + * @param bucket_count Main hash bucket size to use for the new graph context. + * @return c_err_t C_ERR_OK on complete success, C_ERR_PARAM on invalid inputs, or C_ERR_FAIL on structural parsing failures. + */ +c_err_t c_NlpCharGraph_ImportJSON(c_NlpCharGraph_t* self, const char* filepath, c_size_t bucket_count); + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +/** + * @brief 根据当前已出现的字，预测并确定后续最应该输出什么字 + * @param self 指向已完成语料分析的拓扑图实例指针 + * @param current_char 当前最新出现的字（Unicode 码点） + * @param mode 选择的预测模式（贪婪 或 抽样） + * @param out_next_char 接收预测出的下一个字的指针 + * @param out_confidence 接收该预测字的置信度（概率值，0.0 ~ 1.0） + * @return c_err_t C_ERR_OK 表示成功预测，C_ERR_PARAM 表示参数错误，C_ERR_FAIL 表示该字是生僻词/无后续关联 + */ +c_err_t c_NlpCharGraph_PredictNext(c_NlpCharGraph_t* self, c_ucs4_t current_char, c_NlpPredictMode_t mode, + c_ucs4_t* out_next_char, double* out_confidence); + + +#endif /*INCLUDED_C_NLPCHARGRAPH_H*/ diff --git a/cNLP/Library/c_NlpCharGraph.t.c b/cNLP/Library/c_NlpCharGraph.t.c new file mode 100644 index 0000000..724f688 --- /dev/null +++ b/cNLP/Library/c_NlpCharGraph.t.c @@ -0,0 +1,189 @@ +#include "c_NlpCharGraph.h" +#include +#include + +#include "c_StringBuffer.h" +#include "c_utf8_file.h" + +#define RUN_TEST(test_case, name) \ + do { \ + printf("[RUN] %s... ", name); \ + if (test_case) { \ + printf("\033[32mPASSED\033[0m\n"); \ + } else { \ + printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \ + return C_ERR_FAIL; \ + } \ + } while(0) + +static c_err_t test(void) { + /* 29. Arbitrary-Width Character Relation Graph Testing */ + c_NlpCharGraph_t var_graph; + + // Initialize graph with 512 main buckets and an arbitrary custom relation layout size of 128 channels + c_err_t err = c_NlpCharGraph_Init(&var_graph, 512, 128); + RUN_TEST(err == C_ERR_OK, "Variable-width graph initialization returns C_ERR_OK"); + + c_ucs4_t sample_text[] = { 0x540C, 0x6B65, 0x540C }; // '同', '步', '同' + err = c_NlpCharGraph_Process(&var_graph, sample_text, 3); + RUN_TEST(err == C_ERR_OK && var_graph.total_unique == 2, "Processed text array inside flexible capacity container"); + + c_NlpCharNode_t* node_tong = c_NlpCharGraph_GetNode(&var_graph, 0x540C); // '同' + RUN_TEST(node_tong != NULL && node_tong->relation_capacity == 128, "Verified node dynamically inherited the arbitrary 128-bucket relationship size setting"); + RUN_TEST(node_tong->next_chars != NULL && node_tong->prev_chars != NULL, "relational heap allocation arrays successfully bound"); + c_NlpCharGraph_Destroy(&var_graph); + + /* ------------------------------------------------------------------------------------------------------------------ */ + /* */ + + /* 30. Corrected Context Window Proximity Analytical Verification */ + c_NlpCharGraph_t win_graph; + c_NlpCharGraph_Init(&win_graph, 1024, C_UNICODE_MAX); + + // Corpus structural token layout: "A B B C A" -> Test multi-point matching + // Let 0x0041 = 'A', 0x0042 = 'B' + c_ucs4_t cluster_corpus[] = { 0x0041, 0x0042, 0x0042, 0x0043, 0x0041 }; + err = c_NlpCharGraph_Process(&win_graph, cluster_corpus, 5); + RUN_TEST(err == C_ERR_OK, "Cluster corpus parsed inside window context analyzer container"); + + c_size_t min_d = 0; + double avg_d = 0.0; + c_size_t cross_count = 0; + c_size_t count_precedes_1 = 0; + c_size_t count_precedes_2 = 0; + + // Analyze token proximity between 'A' (Index 0, 4) and 'B' (Index 1, 2) with a window threshold size of 3 + // Target matches: + // For A at 0: B at 1 (dist 1, A precedes), B at 2 (dist 2, A precedes) -> 2 pairs + // For A at 4: B at 1 (dist 3, B precedes), B at 2 (dist 2, B precedes) -> 2 pairs + // Total co-occurrences expected = 4 crossings. Min distance = 1. + // Total distance sum = 1 + 2 + 3 + 2 = 8. Avg distance = 8 / 4 = 2.0. + err = c_NlpCharGraph_AnalyzeDistanceWindow(&win_graph, 0x0041, 0x0042, 3, + &min_d, &avg_d, &cross_count, + &count_precedes_1, &count_precedes_2); + + RUN_TEST(err == C_ERR_OK, "Corrected window distance execution completed successfully"); + RUN_TEST(cross_count == 4, "Accurately extracted exactly 4 window crossing combinations across the cluster"); + RUN_TEST(min_d == 1, "Accurately isolated absolute minimum distance down to 1 token ('A' at 0 vs 'B' at 1)"); + RUN_TEST(avg_d == 2.0, "Accurately evaluated average mathematical mean distance at 2.0 tokens precisely"); + RUN_TEST(count_precedes_1 == 2, "Verified 'A' preceded 'B' exactly 2 times across window evaluation frames"); + RUN_TEST(count_precedes_2 == 2, "Verified 'B' preceded 'A' exactly 2 times across window evaluation frames"); + + c_NlpCharGraph_Destroy(&win_graph); + + /* ------------------------------------------------------------------------------------------------------------------ */ + /* */ + + /* 31. Unicode Character Graph JSON Export Verification */ + c_NlpCharGraph_t json_graph; + const char* output_json_path = "nlp_graph_export.json"; + + // Initialize graph context + err = c_NlpCharGraph_Init(&json_graph, 16, 4); + RUN_TEST(err == C_ERR_OK, "JSON Graph context initialized successfully"); + + // Input string: "AI大模型" -> Translated into explicit unicode codepoint stream arrays + c_ucs4_t export_corpus[] = { 0x0041, 0x0049, 0x5927, 0x6A21, 0x578B }; + err = c_NlpCharGraph_Process(&json_graph, export_corpus, 5); + RUN_TEST(err == C_ERR_OK, "Export corpus strings parsed smoothly inside graph matrices"); + + // Execute JSON Export API pipeline + err = c_NlpCharGraph_ExportJSON(&json_graph, output_json_path); + RUN_TEST(err == C_ERR_OK, "c_NlpCharGraph_ExportJSON successfully generated JSON file on disk"); + + // Read file back to verify it contains valid structural characters + c_StringBuffer_t read_verify_sb; + c_StringBuffer_Init(&read_verify_sb, 1024); + err = c_utf8_file_read(output_json_path, &read_verify_sb); + + RUN_TEST(err == C_ERR_OK, "Successfully read generated JSON output file back into workspace"); + RUN_TEST(strstr(read_verify_sb.buffer, "\"total_words\": 5") != NULL, "JSON metadata section successfully parsed"); + RUN_TEST(strstr(read_verify_sb.buffer, "\"char\": \"大\"") != NULL, "Unicode codepoint 0x5927 successfully serialized to UTF-8 character string '大'"); + RUN_TEST(strstr(read_verify_sb.buffer, "\"next_chars\":") != NULL, "Directional adjacency edge transition attributes successfully embedded"); + + // Clean up temporary files and memory structures + c_StringBuffer_Destroy(&read_verify_sb); + c_NlpCharGraph_Destroy(&json_graph); + + /* ------------------------------------------------------------------------------------------------------------------ */ + /* */ + + /* 32. Unicode Character Graph JSON Import Validation */ + c_NlpCharGraph_t imported_graph; + const char* target_json_src = "nlp_graph_export.json"; + + // Dependencies checkpoint: Ensure file exists before running verification passes + // (Generated by calling c_NlpCharGraph_ExportJSON in the previous step) + err = c_NlpCharGraph_ImportJSON(&imported_graph, target_json_src, 1024); + RUN_TEST(err == C_ERR_OK, "c_NlpCharGraph_ImportJSON parsed JSON text stream and built the graph successfully"); + + // Validate Structural Reconstitution Fidelity Metrics + RUN_TEST(imported_graph.total_words == 5, "Reconstructed metadata 'total_words' matches value: 5"); + + // Fetch and check nodes internally + c_NlpCharNode_t* check_da = c_NlpCharGraph_GetNode(&imported_graph, 0x5927); // '大' + RUN_TEST(check_da != NULL, "Unicode Node 0x5927 ('大') successfully verified in bucket storage"); + if (check_da) { + RUN_TEST(check_da->frequency == 1, "Frequency counter holds correct historical state value: 1"); + RUN_TEST(check_da->offset_list.count == 1, "Offsets dynamic list successfully loaded elements"); + + // Trace and check directional bigram linkages + c_size_t min_d = 0; + double avg_d = 0.0; + c_size_t crossings = 0; + c_size_t p1 = 0, p2 = 0; + err = c_NlpCharGraph_AnalyzeDistanceWindow(&imported_graph, 0x5927, 0x6A21, 2, &min_d, &avg_d, &crossings, &p1, &p2); // '大' vs '模' + RUN_TEST(err == C_ERR_OK && crossings == 1, "Relational bigram matching layers successfully operational post-import"); + } + + c_NlpCharGraph_Destroy(&imported_graph); + + // remove(output_json_path); // Remove transient file from disk space + + /* ------------------------------------------------------------------------------------------------------------------ */ + /* */ + + /* 31. 语言模型后续字预测验证 (c_NlpCharGraph_PredictNext) */ + c_NlpCharGraph_t pred_graph; + c_NlpCharGraph_Init(&pred_graph, 1024, 16); + + // 载入高重复、高确定性的测试语料："大模型大模型大项目" + // '大' 后面出现了 2 次 '模'，1 次 '项'。总后续数 = 3 + c_ucs4_t train_data[] = { + 0x5927, 0x6A21, 0x578B, // 大模型 + 0x5927, 0x6A21, 0x578B, // 大模型 + 0x5927, 0x9879, 0x76EE // 大项目 + }; + err = c_NlpCharGraph_Process(&pred_graph, train_data, 9); + RUN_TEST(err == C_ERR_OK, "Prediction training data parsed smoothly"); + + c_ucs4_t predicted_char = 0; + double confidence = 0.0; + + // 进行确定性的贪婪模式预测：输入 '大' (0x5927) + // 预期：'模' (0x6A21) 出现的概率是 2/3 = 66.67%，远高于 '项' 的 1/3 + err = c_NlpCharGraph_PredictNext(&pred_graph, 0x5927, C_PREDICT_MODE_GREEDY, &predicted_char, &confidence); + + RUN_TEST(err == C_ERR_OK, "PredictNext completed execution path successfully"); + RUN_TEST(predicted_char == 0x6A21, "Greedy mode correctly deterministically chooses '模' following '大'"); + // 校验置信度 2.0 / 3.0 近似等于 0.666667 + RUN_TEST(confidence > 0.66 && confidence < 0.67, "Confidence value evaluated exactly at 66.67%%"); + + // 测试生僻词/无后续关联的边界安全 + err = c_NlpCharGraph_PredictNext(&pred_graph, 0x76EE, C_PREDICT_MODE_GREEDY, &predicted_char, &confidence); // '目' 是文章末尾 + RUN_TEST(err == C_ERR_FAIL, "PredictNext safely rejects character with zero trailing context links"); + + c_NlpCharGraph_Destroy(&pred_graph); + + + /* ------------------------------------------------------------------------------------------------------------------ */ + /* */ + + return C_ERR_OK; +} + +int main(int argc, char** argv){ + return test(); + + return 0; +} diff --git a/cNLP/Library/c_NlpLanguageModel.c b/cNLP/Library/c_NlpLanguageModel.c new file mode 100644 index 0000000..f74ff24 --- /dev/null +++ b/cNLP/Library/c_NlpLanguageModel.c @@ -0,0 +1,94 @@ +#include +#include + + +c_err_t c_NlpLanguageModel_Init(c_NlpLanguageModel_t* self, c_size_t max_n, c_size_t bucket_count) { + if (!self || max_n < 2 || bucket_count == 0) return C_ERR_PARAM; + + self->max_n = max_n; + self->backoff_alpha = 0.4; // Industrial standard alpha decay baseline configuration + + // Allocate space for all N-gram tiers down to 2-gram (size = max_n - 1) + c_size_t graph_count = max_n - 1; + self->levels = (c_NlpNgramGraph_t*)C_ALLOC(sizeof(c_NlpNgramGraph_t) * graph_count); + if (!self->levels) return C_ERR_NOMEM; + + // Initialize sub-graphs from highest order down to lowest (Bigram) + for (c_size_t i = 0; i < graph_count; i++) { + c_size_t current_n = max_n - i; + c_err_t err = c_NlpNgramGraph_Init(&self->levels[i], current_n, bucket_count, 16); + if (err != C_ERR_OK) { + // Rollback previously initialized sub-graphs on allocation failure + for (c_size_t k = 0; k < i; k++) { + c_NlpNgramGraph_Destroy(&self->levels[k]); + } + C_FREE(self->levels); + self->levels = NULL; + return err; + } + } + return C_ERR_OK; +} + +c_err_t c_NlpLanguageModel_Train(c_NlpLanguageModel_t* self, const c_ucs4_t* unicode_array, c_size_t length) { + if (!self || !self->levels || !unicode_array || length == 0) return C_ERR_PARAM; + + c_size_t graph_count = self->max_n - 1; + // Feed data to train all nested sub-graphs simultaneously + for (c_size_t i = 0; i < graph_count; i++) { + c_err_t err = c_NlpNgramGraph_Process(&self->levels[i], unicode_array, length); + if (err != C_ERR_OK) return err; + } + return C_ERR_OK; +} + +c_err_t c_NlpLanguageModel_PredictSmooth(c_NlpLanguageModel_t* self, const c_ucs4_t* history_context, c_size_t history_len, + c_ucs4_t* out_next_char, double* out_confidence) { + if (!self || !self->levels || !history_context || history_len == 0 || !out_next_char || !out_confidence) { + return C_ERR_PARAM; + } + + c_size_t graph_count = self->max_n - 1; + double current_alpha_multiplier = 1.0; + + // Step down from the highest-order N-gram down to the baseline Bigram level + for (c_size_t i = 0; i < graph_count; i++) { + c_NlpNgramGraph_t* current_graph = &self->levels[i]; + c_size_t expected_history_len = current_graph->n_value - 1; + + // If our current available history sequence matches the tier's required context length + if (history_len >= expected_history_len) { + // Extract a sliced pointer matching exactly the trailing edge suffix of our history + c_size_t offset_start = history_len - expected_history_len; + const c_ucs4_t* sliced_history = &history_context[offset_start]; + + double baseline_confidence = 0.0; + // Attempt a retrieval match execution path + c_err_t err = c_NlpNgramGraph_PredictNext(current_graph, sliced_history, out_next_char, &baseline_confidence); + + if (err == C_ERR_OK) { + // High-order match successful! Deduct confidence with our decay penalty multipliers + *out_confidence = baseline_confidence * current_alpha_multiplier; + return C_ERR_OK; + } + } + + // If the query falls through (C_ERR_FAIL), cascade lower and apply the alpha smoothing penalty + current_alpha_multiplier *= self->backoff_alpha; + } + + return C_ERR_FAIL; // Completely out of vocabulary context indicators across all internal graphs +} + +void c_NlpLanguageModel_Destroy(c_NlpLanguageModel_t* self) { + if (!self || !self->levels) return; + + c_size_t graph_count = self->max_n - 1; + for (c_size_t i = 0; i < graph_count; i++) { + c_NlpNgramGraph_Destroy(&self->levels[i]); + } + C_FREE(self->levels); + self->levels = NULL; + self->max_n = 0; +} + diff --git a/cNLP/Library/c_NlpLanguageModel.h b/cNLP/Library/c_NlpLanguageModel.h new file mode 100644 index 0000000..83c8426 --- /dev/null +++ b/cNLP/Library/c_NlpLanguageModel.h @@ -0,0 +1,38 @@ +#ifndef INCLUDED_C_NLPLANGUAGEMODEL_H +#define INCLUDED_C_NLPLANGUAGEMODEL_H + +#ifndef INCLUDED_C_NLPNGRAMGRAPH_H +#include +#endif /*INCLUDED_C_NLPNGRAMGRAPH_H*/ + + +typedef struct { + c_NlpNgramGraph_t* levels; /* Array of N-gram graphs sorted from highest order down to 2-gram */ + c_size_t max_n; /* The maximum configuration order N of the model (e.g., 3 for Trigram) */ + double backoff_alpha; /* Proportional smoothing decay factor (typically 0.4) used per back-off step */ +} c_NlpLanguageModel_t; + +/** + * @brief Initializes a multi-tier cascaded Language Model environment supporting back-off smoothing. + * @param self Pointer to the uninitialized language model structure. + * @param max_n Maximum model order threshold (e.g., 3 creates a Trigram + Bigram cascading mesh). + * @param bucket_count Main hash allocation array size mapped down to sub-graphs. + * @return c_err_t C_ERR_OK on successful initialization, or dynamic memory OOM codes. + */ +c_err_t c_NlpLanguageModel_Init(c_NlpLanguageModel_t* self, c_size_t max_n, c_size_t bucket_count); + +/** + * @brief Core Prediction: Evaluates histories and executes automated back-off smoothing cascades. + */ +c_err_t c_NlpLanguageModel_PredictSmooth(c_NlpLanguageModel_t* self, const c_ucs4_t* history_context, c_size_t history_len, + c_ucs4_t* out_next_char, double* out_confidence); + +/** + * @brief Destroys and flushes all cascaded hierarchy sub-graphs completely. + */ +void c_NlpLanguageModel_Destroy(c_NlpLanguageModel_t* self); + + +c_err_t c_NlpLanguageModel_Train(c_NlpLanguageModel_t* self, const c_ucs4_t* unicode_array, c_size_t length); + +#endif /*INCLUDED_C_NLPLANGUAGEMODEL_H*/ diff --git a/cNLP/Library/c_NlpLanguageModel.t.c b/cNLP/Library/c_NlpLanguageModel.t.c new file mode 100644 index 0000000..9d856fd --- /dev/null +++ b/cNLP/Library/c_NlpLanguageModel.t.c @@ -0,0 +1,58 @@ +#include "c_NlpLanguageModel.h" +#include +#include + +#define RUN_TEST(test_case, name) \ + do { \ + printf("[RUN] %s... ", name); \ + if (test_case) { \ + printf("\033[32mPASSED\033[0m\n"); \ + } else { \ + printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \ + return C_ERR_FAIL; \ + } \ + } while(0) + +static c_err_t test(void) { + /* 34. Katz Back-off Smoothing Cascade Verification */ + c_NlpLanguageModel_t lm; + // Instantiate a 3-tier system (Trigram + Bigram cascaded backend framework mesh) + c_err_t err = c_NlpLanguageModel_Init(&lm, 3, 512); + RUN_TEST(err == C_ERR_OK, "Cascaded Language Model successfully initialized"); + + // Training set containing: "大模型" and "微模型" + c_ucs4_t text_stream[] = { + 0x5927, 0x6A21, 0x578B, /* 大模型 */ + 0x5FAE, 0x6A21, 0x578B /* 微模型 */ + }; + err = c_NlpLanguageModel_Train(&lm, text_stream, 6); + RUN_TEST(err == C_ERR_OK, "Cascaded models successfully ingested corpus dataset streams"); + + c_ucs4_t next_prediction = 0; + double confidence_score = 0.0; + + // --- Scenario A: Perfect high-order match --- + // Query context: "大模" -> Expected result: "型" (100% confidence from Trigram tier) + c_ucs4_t perfect_history[] = { 0x5927, 0x6A21 }; + err = c_NlpLanguageModel_PredictSmooth(&lm, perfect_history, 2, &next_prediction, &confidence_score); + RUN_TEST(err == C_ERR_OK && next_prediction == 0x578B, "Scenario A: High-order match correctly resolves to '型'"); + RUN_TEST(confidence_score == 1.0, "Scenario A: Direct hit returns full unpenalized confidence metrics"); + + // --- Scenario B: Unseen Trigram history forces a Back-off step --- + // Query context: "新模" -> The 3-gram "新模" is completely unseen in our training text + // Expected behavior: Trigram lookup fails. Model smoothly backs off to the Bigram tier matching only "模" -> "型". + // Mathematical projection: 1.0 baseline confidence * 0.4 alpha penalty = 0.4 adjusted score. + c_ucs4_t unseen_trigram_history[] = { 0x65B0, 0x6A21 }; // "新模" + err = c_NlpLanguageModel_PredictSmooth(&lm, unseen_trigram_history, 2, &next_prediction, &confidence_score); + + RUN_TEST(err == C_ERR_OK, "Scenario B: Back-off routine successfully processes the unseen 3-gram phrase"); + RUN_TEST(next_prediction == 0x578B, "Scenario B: Lower-order bigram successfully resolves '模' ➔ '型'"); + RUN_TEST(confidence_score == 0.4, "Scenario B: Confidence safely records the 0.4 alpha smoothing decay penalty"); + + c_NlpLanguageModel_Destroy(&lm); + return C_ERR_OK; +} + +int main(int argc, char** argv){ + return test(); +} diff --git a/cNLP/Library/c_NlpNgramGraph.c b/cNLP/Library/c_NlpNgramGraph.c new file mode 100644 index 0000000..6b2fc0d --- /dev/null +++ b/cNLP/Library/c_NlpNgramGraph.c @@ -0,0 +1,242 @@ +#include +#include + +#include +#include + +/* FNV-1a 32位 滾動哈希，用於對變長 Unicode 陣列計算主桶索引 */ +C_STATIC_FORCE_INLINE +c_size_t c_NlpNgram_HashContext(const c_ucs4_t* context, c_size_t len, c_size_t bucket_count) { + unsigned int hash = 2166136261U; + for (c_size_t i = 0; i < len; i++) { + hash ^= context[i]; + hash *= 16777619U; + } + return (c_size_t)(hash % bucket_count); +} + +/* 比較兩個上下文陣列是否完全一致 */ +C_STATIC_FORCE_INLINE +c_bool_t c_NlpNgram_ContextEquals(const c_ucs4_t* ctx1, const c_ucs4_t* ctx2, c_size_t len) { + for (c_size_t i = 0; i < len; i++) { + if (ctx1[i] != ctx2[i]) return C_FALSE; + } + return C_TRUE; +} + +/* 建立並初始化一個全新的歷史狀態節點 */ +C_STATIC_FORCE_INLINE +c_NgramNode_t* c_NgramNode_Create(const c_ucs4_t* context, c_size_t ctx_len, c_size_t rel_cap) { + c_NgramNode_t* node = (c_NgramNode_t*)C_ALLOC(sizeof(c_NgramNode_t)); + if (!node) return NULL; + + node->frequency = 0; + node->relation_capacity = rel_cap; + node->next = NULL; + + // 深拷貝上下文狀態陣列 + node->context = (c_ucs4_t*)C_ALLOC(sizeof(c_ucs4_t) * ctx_len); + node->next_chars = (c_NgramEdge_t**)C_ALLOC(sizeof(c_NgramEdge_t*) * rel_cap); + + if (!node->context || !node->next_chars) { + C_FREE(node->context); + C_FREE(node->next_chars); + C_FREE(node); + return NULL; + } + + memcpy(node->context, context, sizeof(c_ucs4_t) * ctx_len); + for (c_size_t i = 0; i < rel_cap; i++) { + node->next_chars[i] = NULL; + } + return node; +} + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +c_err_t c_NlpNgramGraph_Init(c_NlpNgramGraph_t* self, c_size_t n_value, c_size_t bucket_count, c_size_t relation_default_capacity) { + if (!self || n_value < 2 || bucket_count == 0 || relation_default_capacity == 0) return C_ERR_PARAM; + + self->n_value = n_value; + self->bucket_count = bucket_count; + self->relation_default_capacity = relation_default_capacity; + self->total_states = 0; + + self->buckets = (c_NgramNode_t**)C_ALLOC(sizeof(c_NgramNode_t*) * bucket_count); + if (!self->buckets) return C_ERR_NOMEM; + + for (c_size_t i = 0; i < bucket_count; i++) { + self->buckets[i] = NULL; + } + return C_ERR_OK; +} + +/* 拉鏈法安全獲取或建立上下文狀態節點 */ +static c_NgramNode_t* c_NlpNgramGraph_GetOrCreate(c_NlpNgramGraph_t* self, const c_ucs4_t* context) { + c_size_t ctx_len = self->n_value - 1; + c_size_t idx = c_NlpNgram_HashContext(context, ctx_len, self->bucket_count); + + c_NgramNode_t* curr = self->buckets[idx]; + c_NgramNode_t* prev = NULL; + + while (curr != NULL) { + if (c_NlpNgram_ContextEquals(curr->context, context, ctx_len)) { + return curr; // 命中已有歷史狀態 + } + prev = curr; + curr = curr->next; + } + + // 未命中，建立新歷史狀態節點 + c_NgramNode_t* new_node = c_NgramNode_Create(context, ctx_len, self->relation_default_capacity); + if (!new_node) return NULL; + + if (prev == NULL) { + self->buckets[idx] = new_node; + } else { + prev->next = new_node; + } + self->total_states++; + return new_node; +} + +/* 核心注入：滑動視窗掃描整篇文章語料 */ +c_err_t c_NlpNgramGraph_Process(c_NlpNgramGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length) { + if (!self || !self->buckets || !unicode_array || length == 0) return C_ERR_PARAM; + + c_size_t n = self->n_value; + // 如果文章字數小於 N 元模型所需的長度，無法建立狀態 + if (length < n) return C_ERR_OK; + + // 滑動視窗上界為 length - n + 1 + for (c_size_t i = 0; i <= length - n; i++) { + // 1. 當前滑動視窗的起點指標，指向長度為 N-1 的歷史上下文 + const c_ucs4_t* current_context = &unicode_array[i]; + + // 2. 獲取或建立對應的多元歷史節點 + c_NgramNode_t* node = c_NlpNgramGraph_GetOrCreate(self, current_context); + if (!node) return C_ERR_NOMEM; + + node->frequency++; + + // 3. 提取第 N 個字作為預測轉移目標 + c_ucs4_t target_char = unicode_array[i + n - 1]; + + // 4. 將預測目標掛入該歷史節點的 next_chars 邊哈希拉鏈中 + c_size_t edge_idx = (c_size_t)(target_char % node->relation_capacity); + c_NgramEdge_t* edge = node->next_chars[edge_idx]; + c_bool_t edge_found = C_FALSE; + + while (edge != NULL) { + if (edge->target_cp == target_char) { + edge->co_count++; + edge_found = C_TRUE; + break; + } + edge = edge->next; + } + + if (!edge_found) { + c_NgramEdge_t* new_edge = (c_NgramEdge_t*)C_ALLOC(sizeof(c_NgramEdge_t)); + if (!new_edge) return C_ERR_NOMEM; + new_edge->target_cp = target_char; + new_edge->co_count = 1; + new_edge->next = node->next_chars[edge_idx]; + node->next_chars[edge_idx] = new_edge; + } + } + return C_ERR_OK; +} + +void c_NlpNgramGraph_Destroy(c_NlpNgramGraph_t* self) { + if (!self || !self->buckets || self->bucket_count == 0) return; + + c_NgramNode_t** local_buckets = self->buckets; + c_size_t local_count = self->bucket_count; + + // 搶先阻斷外部二次訪問漏洞 + self->buckets = NULL; + self->bucket_count = 0; + self->total_states = 0; + + for (c_size_t i = 0; i < local_count; i++) { + c_NgramNode_t* node = local_buckets[i]; + + while (node != NULL) { + c_NgramNode_t* next_node = node->next; + + // 1. 釋放深層歷史上下文陣列 + if (node->context != NULL) { + C_FREE(node->context); + } + + // 2. 釋放後續預測鄰接邊拉鏈 + if (node->next_chars != NULL) { + for (c_size_t k = 0; k < node->relation_capacity; k++) { + c_NgramEdge_t* edge = node->next_chars[k]; + while (edge != NULL) { + c_NgramEdge_t* temp_edge = edge->next; + C_FREE(edge); + edge = temp_edge; + } + } + C_FREE(node->next_chars); + } + + // 3. 釋放狀態節點本身 + C_FREE(node); + node = next_node; + } + } + C_FREE(local_buckets); +} + +c_err_t c_NlpNgramGraph_PredictNext(c_NlpNgramGraph_t* self, const c_ucs4_t* history_context, + c_ucs4_t* out_next_char, double* out_confidence) { + if (!self || !self->buckets || !history_context || !out_next_char || !out_confidence) { + return C_ERR_PARAM; + } + + c_size_t ctx_len = self->n_value - 1; + c_size_t idx = c_NlpNgram_HashContext(history_context, ctx_len, self->bucket_count); + + // 1. 定位多元歷史狀態 + c_NgramNode_t* node = self->buckets[idx]; + while (node != NULL) { + if (c_NlpNgram_ContextEquals(node->context, history_context, ctx_len)) { + break; + } + node = node->next; + } + + // 如果語料庫中從未出現過這段連續的歷史短语（未命中） + if (!node || node->frequency == 0) { + return C_ERR_FAIL; + } + + // 2. 統計該狀態下的邊權重，找出概率最高的轉移字符 + c_size_t total_co_count = 0; + c_size_t max_co_count = 0; + c_ucs4_t best_target_cp = 0; + + for (c_size_t i = 0; i < node->relation_capacity; i++) { + c_NgramEdge_t* edge = node->next_chars[i]; + while (edge != NULL) { + total_co_count += edge->co_count; + if (edge->co_count > max_co_count) { + max_co_count = edge->co_count; + best_target_cp = edge->target_cp; + } + edge = edge->next; + } + } + + if (total_co_count == 0) return C_ERR_FAIL; + + // 输出預測最優解與馬爾可夫最大似然概率 + *out_next_char = best_target_cp; + *out_confidence = (double)max_co_count / (double)total_co_count; + + return C_ERR_OK; +} diff --git a/cNLP/Library/c_NlpNgramGraph.h b/cNLP/Library/c_NlpNgramGraph.h new file mode 100644 index 0000000..8195691 --- /dev/null +++ b/cNLP/Library/c_NlpNgramGraph.h @@ -0,0 +1,51 @@ +#ifndef INCLUDED_C_NLPNGRAMGRAPH_H +#define INCLUDED_C_NLPNGRAMGRAPH_H + +#ifndef INCLUDED_C_UTF8_H +#include +#endif /*INCLUDED_C_UTF8_H*/ + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + + +/* --- 1. N-gram 預測邊（記錄後續可能出現的單個字及頻次） --- */ +typedef struct c_NgramEdge_t { + c_ucs4_t target_cp; /* 後續預測出的第 N 個字 */ + c_size_t co_count; /* 該字在當前上下文後出現的次數 */ + struct c_NgramEdge_t* next; /* 鄰接邊哈希衝突拉鏈 */ +} c_NgramEdge_t; + +/* --- 2. N-gram 上下文節點（代表長度為 N-1 的歷史狀態） --- */ +typedef struct c_NgramNode_t { + c_ucs4_t* context; /* 歷史上下文陣列，長度為 N-1 */ + c_size_t frequency; /* 該上下文總出現次數 */ + + c_NgramEdge_t** next_chars; /* 後續字哈希桶（動態分配） */ + c_size_t relation_capacity; /* 邊哈希桶容量 */ + + struct c_NgramNode_t* next; /* 主哈希表衝突拉鏈指针（解決主桶碰撞） */ +} c_NgramNode_t; + +/* --- 3. 多元馬爾可夫鏈模型主控結構体 --- */ +typedef struct { + c_NgramNode_t** buckets; /* 主哈希表（儲存所有不重複的歷史上下文狀態） */ + c_size_t bucket_count; /* 主哈希桶大小 */ + c_size_t n_value; /* 模型階數 N (例如：3 代表 Trigram 模型) */ + c_size_t relation_default_capacity; /* 子節點邊哈希桶默認容量 */ + c_size_t total_states; /* 記錄圖中唯一存在的歷史狀態總數 */ +} c_NlpNgramGraph_t; + +/* ------------------------------------------------------------------------------------------------------------------ */ +/* */ + +c_err_t c_NlpNgramGraph_Init(c_NlpNgramGraph_t* self, c_size_t n_value, c_size_t bucket_count, c_size_t relation_default_capacity); + +void c_NlpNgramGraph_Destroy(c_NlpNgramGraph_t* self); + +c_err_t c_NlpNgramGraph_Process(c_NlpNgramGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length); + +c_err_t c_NlpNgramGraph_PredictNext(c_NlpNgramGraph_t* self, const c_ucs4_t* history_context, + c_ucs4_t* out_next_char, double* out_confidence); + +#endif /*INCLUDED_C_NLPNGRAMGRAPH_H*/ diff --git a/cNLP/Library/c_NlpNgramGraph.t.c b/cNLP/Library/c_NlpNgramGraph.t.c new file mode 100644 index 0000000..75b89b1 --- /dev/null +++ b/cNLP/Library/c_NlpNgramGraph.t.c @@ -0,0 +1,52 @@ +#include "c_NlpNgramGraph.h" +#include +#include + +#define RUN_TEST(test_case, name) \ + do { \ + printf("[RUN] %s... ", name); \ + if (test_case) { \ + printf("\033[32mPASSED\033[0m\n"); \ + } else { \ + printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \ + return C_ERR_FAIL; \ + } \ + } while(0) + +static c_err_t test(void) { + /* 33. 多元馬爾可夫鏈（Trigram）核心通路驗證 */ + c_NlpNgramGraph_t trigram; + // 建立 3元 模型：需要根據 2 个字的历史預測第 3 个字 + c_err_t err = c_NlpNgramGraph_Init(&trigram, 3, 512, 8); + RUN_TEST(err == C_ERR_OK, "Trigram Graph initialized cleanly"); + + // 載入預測語料："自然语言、自然界、自然语言" + // 當已知歷史短语是 "自然" 时，"语" 出现了 2 次，"界" 出现了 1 次 + c_ucs4_t corpus_stream[] = { + 0x81EA, 0x7136, 0x8BED, 0x8A00, // 自然语言 + 0x81EA, 0x7136, 0x754C, // 自然界 + 0x81EA, 0x7136, 0x8BED, 0x8A00 // 自然语言 + }; + err = c_NlpNgramGraph_Process(&trigram, corpus_stream, 11); + RUN_TEST(err == C_ERR_OK && trigram.total_states > 0, "Trigram processed training data stream"); + + // 構建查詢歷史：[0x81EA, 0x7136] ➔ 代表 "自然" + c_ucs4_t query_history[] = { 0x81EA, 0x7136 }; + c_ucs4_t pred_res = 0; + double pred_conf = 0.0; + + err = c_NlpNgramGraph_PredictNext(&trigram, query_history, &pred_res, &pred_conf); + RUN_TEST(err == C_ERR_OK, "PredictNext completed high-order context traversal paths"); + + // 預期：準確命中 "语" (0x8BED)，置信度為 2 / 3 = 66.67% + RUN_TEST(pred_res == 0x8BED, "Trigram language model predicts '语' given advanced context '自然'"); + RUN_TEST(pred_conf > 0.66 && pred_conf < 0.67, "Context-bound model confidence targets 66.67%% correctly"); + + c_NlpNgramGraph_Destroy(&trigram); + + return C_ERR_OK; +} + +int main(int argc, char** argv){ + return test(); +}