pchen/cAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

c_NlpCharGraph： 分析一篇文章，建立字与字的关系

Browse Source 
c_NlpNgramGraph: 基于 n 个已经出现的词，预估后续可能出现的词
c_NlpLanguageModel: 如果有的词组合形式没有出现过，找到之前学习过最长类似的词句进行预测
This commit is contained in:
pchen
2026-08-10 03:12:45 +08:00
parent e45398991f
commit e6a29bcdfd
10 changed files with 1560 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cKit/Foundation/c_utf8.h
+16
View File
@@ -5,6 +5,14 @@
#include <c_Base.h>
#endif /*INCLUDED_C_BASE_H*/
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
#ifndef C_UNICODE_MAX
#define C_UNICODE_MAX 0x10FFFFU
#endif
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
@@ -15,6 +23,14 @@ typedef uint16_t c_uint16_t;
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
C_STATIC_FORCE_INLINE
bool c_is_valid_unicode(uint32_t code) {
// Unicode 码点不能超过 0x10FFFF，且必须排除 UTF-16 代理对范围 (0xD800 ~ 0xDFFF)
if (code > C_UNICODE_MAX) return false;
if (code >= 0xD800 && code <= 0xDFFF) return false;
return true;
}
/**
* @brief 获取一个 UTF-8 字符在当前指针位置所占用的实际字节数 (1 ~ 4 字节)
*/
cNLP/Library/c_NlpCharGraph.c
+704
View File
@@ -0,0 +1,704 @@
#include <c_NlpCharGraph.h>
#include <c_Memory.h>
#include <stdio.h>
#include "c_StringBuffer.h"
#include "c_utf8_file.h"
C_STATIC_FORCE_INLINE
c_size_t c_NlpGraph_Hash(c_ucs4_t cp, c_size_t capacity) {
return (c_size_t)(cp % capacity);
}
C_STATIC_FORCE_INLINE
c_err_t c_NlpOffsetList_Init(c_NlpOffsetList_t* list) {
list->count = 0;
list->capacity = 4;
list->offsets = (c_size_t*)C_ALLOC(sizeof(c_size_t) * list->capacity);
return list->offsets ? C_ERR_OK : C_ERR_NOMEM;
}
static c_err_t c_NlpOffsetList_Add(c_NlpOffsetList_t* list, c_size_t offset) {
if (list->count >= list->capacity) {
c_size_t new_cap = list->capacity * 2;
c_size_t* new_offsets = (c_size_t*)C_ALLOC(sizeof(c_size_t) * new_cap);
if (!new_offsets) return C_ERR_NOMEM;
memcpy(new_offsets, list->offsets, sizeof(c_size_t) * list->count);
C_FREE(list->offsets);
list->offsets = new_offsets;
list->capacity = new_cap;
}
list->offsets[list->count++] = offset;
return C_ERR_OK;
}
/* --- Dynamic Node Instantiation Engine --- */
static c_NlpCharNode_t* c_NlpCharNode_Create(c_ucs4_t cp, c_size_t relation_capacity) {
if (relation_capacity == 0) return NULL;
c_NlpCharNode_t* node = (c_NlpCharNode_t*)C_ALLOC(sizeof(c_NlpCharNode_t));
if (!node) return NULL;
node->codepoint = cp;
node->frequency = 0;
node->relation_capacity = relation_capacity;
if (c_NlpOffsetList_Init(&node->offset_list) != C_ERR_OK) {
C_FREE(node);
return NULL;
}
// Allocate variable-width hash array tables on the heap
node->next_chars = (c_NlpCharEdge_t**)C_ALLOC(sizeof(c_NlpCharEdge_t*) * relation_capacity);
node->prev_chars = (c_NlpCharEdge_t**)C_ALLOC(sizeof(c_NlpCharEdge_t*) * relation_capacity);
if (!node->next_chars || !node->prev_chars) {
C_FREE(node->next_chars);
C_FREE(node->prev_chars);
C_FREE(node->offset_list.offsets);
C_FREE(node);
return NULL;
}
// Zero out pointers across all newly configured custom slots
for (c_size_t i = 0; i < relation_capacity; i++) {
node->next_chars[i] = NULL;
node->prev_chars[i] = NULL;
}
return node;
}
static c_NlpCharNode_t* c_NlpCharGraph_GetOrCreateNode(c_NlpCharGraph_t* self, c_ucs4_t cp) {
c_size_t idx = c_NlpGraph_Hash(cp, self->bucket_count);
c_NlpCharNode_t* curr = self->buckets[idx];
c_NlpCharNode_t* prev = NULL;
// 沿拉链向下查找匹配的码点
while (curr != NULL) {
if (curr->codepoint == cp) {
return curr; // 完美命中已有节点
}
prev = curr;
curr = curr->next;
}
// 未命中冲突，创建全新节点并通过头插法/尾插法挂入拉链
c_NlpCharNode_t* new_node = c_NlpCharNode_Create(cp, self->relation_default_capacity);
if (!new_node) return NULL;
new_node->next = NULL;
if (prev == NULL) {
self->buckets[idx] = new_node; // 桶内第一个节点
} else {
prev->next = new_node; // 挂在冲突链条末尾
}
self->total_unique++;
return new_node;
}
static c_err_t c_NlpCharNode_AddRelation(c_NlpCharEdge_t** edge_buckets, c_size_t capacity, c_ucs4_t target_cp) {
c_size_t idx = c_NlpGraph_Hash(target_cp, capacity);
c_NlpCharEdge_t* curr = edge_buckets[idx];
while (curr != NULL) {
if (curr->target_cp == target_cp) {
curr->co_count++;
return C_ERR_OK;
}
curr = curr->next;
}
c_NlpCharEdge_t* new_edge = (c_NlpCharEdge_t*)C_ALLOC(sizeof(c_NlpCharEdge_t));
if (!new_edge) return C_ERR_NOMEM;
new_edge->target_cp = target_cp;
new_edge->co_count = 1;
new_edge->next = edge_buckets[idx];
edge_buckets[idx] = new_edge;
return C_ERR_OK;
}
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpCharGraph_Init(c_NlpCharGraph_t* self, c_size_t bucket_count, c_size_t relation_default_capacity) {
if (!self || bucket_count == 0 || relation_default_capacity == 0) return C_ERR_PARAM;
self->bucket_count = bucket_count;
self->relation_default_capacity = relation_default_capacity;
self->total_unique = 0;
self->total_words = 0;
self->buckets = (c_NlpCharNode_t**)C_ALLOC(sizeof(c_NlpCharNode_t*) * bucket_count);
if (!self->buckets) return C_ERR_NOMEM;
for (c_size_t i = 0; i < bucket_count; i++) {
self->buckets[i] = NULL;
}
return C_ERR_OK;
}
c_NlpCharNode_t* c_NlpCharGraph_GetNode(c_NlpCharGraph_t* self, c_ucs4_t cp) {
if (!self || !self->buckets) return NULL;
c_size_t idx = c_NlpGraph_Hash(cp, self->bucket_count);
c_NlpCharNode_t* curr = self->buckets[idx];
while (curr != NULL) {
if (curr->codepoint == cp) return curr;
curr = curr->next;
}
return NULL;
}
c_err_t c_NlpCharGraph_Process(c_NlpCharGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length) {
if (!self || !self->buckets || !unicode_array || length == 0) return C_ERR_PARAM;
self->total_words += length;
for (c_size_t i = 0; i < length; i++) {
c_ucs4_t curr_cp = unicode_array[i];
// 1. 通过拉链机制获取/创建当前节点
c_NlpCharNode_t* curr_node = c_NlpCharGraph_GetOrCreateNode(self, curr_cp);
if (!curr_node) return C_ERR_NOMEM;
curr_node->frequency++;
c_err_t err = c_NlpOffsetList_Add(&curr_node->offset_list, i);
if (err != C_ERR_OK) return err;
// 2. 建立前序边，同时保证前序字在拉链表中不丢失
if (i > 0) {
c_ucs4_t prev_cp = unicode_array[i - 1];
c_NlpCharNode_t* p_node = c_NlpCharGraph_GetOrCreateNode(self, prev_cp);
if (!p_node) return C_ERR_NOMEM;
err = c_NlpCharNode_AddRelation(curr_node->prev_chars, curr_node->relation_capacity, prev_cp);
if (err != C_ERR_OK) return err;
}
// 3. 建立后续边（如“大”关联“模”），同时通过拉链确保“模”字被注册激活
if (i < length - 1) {
c_ucs4_t next_cp = unicode_array[i + 1];
c_NlpCharNode_t* n_node = c_NlpCharGraph_GetOrCreateNode(self, next_cp);
if (!n_node) return C_ERR_NOMEM;
err = c_NlpCharNode_AddRelation(curr_node->next_chars, curr_node->relation_capacity, next_cp);
if (err != C_ERR_OK) return err;
}
}
return C_ERR_OK;
}
static void c_NlpCharEdge_FreeBuckets(c_NlpCharEdge_t** edge_buckets, c_size_t capacity) {
if (!edge_buckets) return;
for (c_size_t i = 0; i < capacity; i++) {
c_NlpCharEdge_t* curr = edge_buckets[i];
while (curr != NULL) {
c_NlpCharEdge_t* temp = curr->next;
C_FREE(curr);
curr = temp;
}
}
}
void c_NlpCharGraph_Destroy(c_NlpCharGraph_t* self) {
if (!self || !self->buckets || self->bucket_count==0) return;
for (c_size_t i = 0; i < self->bucket_count; i++) {
c_NlpCharNode_t* node = self->buckets[i];
// 沿拉链释放所有冲突节点的深层堆空间
while (node != NULL) {
c_NlpCharNode_t* next_node = node->next; // 暂存下一个指针
C_FREE(node->offset_list.offsets);
c_NlpCharEdge_FreeBuckets(node->next_chars, node->relation_capacity);
c_NlpCharEdge_FreeBuckets(node->prev_chars, node->relation_capacity);
C_FREE(node->next_chars);
C_FREE(node->prev_chars);
C_FREE(node);
node = next_node;
}
self->buckets[i] = NULL;
}
C_FREE(self->buckets);
self->bucket_count = 0;
self->total_unique = 0;
self->total_words = 0;
}
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpCharGraph_AnalyzeDistanceWindow(c_NlpCharGraph_t* self, c_ucs4_t cp1, c_ucs4_t cp2, c_size_t window_size,
c_size_t* out_min_dist, double* out_avg_dist, c_size_t* out_co_occurrences,
c_size_t* out_cp1_precedes_count, c_size_t* out_cp2_precedes_count) {
// 1. Defend against parameter null configurations or invalid window constraints
if (!self || !out_min_dist || !out_avg_dist || !out_co_occurrences ||
!out_cp1_precedes_count || !out_cp2_precedes_count || window_size == 0) {
return C_ERR_PARAM;
}
// Initialize all metrics to zero out out-parameters safely
*out_min_dist = (c_size_t)-1;
*out_avg_dist = 0.0;
*out_co_occurrences = 0;
*out_cp1_precedes_count = 0;
*out_cp2_precedes_count = 0;
// 2. Fetch the structural node contexts from the master hash bucket allocations
c_NlpCharNode_t* n1 = c_NlpCharGraph_GetNode(self, cp1);
c_NlpCharNode_t* n2 = c_NlpCharGraph_GetNode(self, cp2);
if (!n1 || !n2 || n1->offset_list.count == 0 || n2->offset_list.count == 0) {
return C_ERR_OK; // Return cleanly with zeroed structures if any token is absent
}
c_size_t* o1 = n1->offset_list.offsets;
c_size_t* o2 = n2->offset_list.offsets;
c_size_t len1 = n1->offset_list.count;
c_size_t len2 = n2->offset_list.count;
c_size_t min_dist = (c_size_t)-1;
double total_dist_sum = 0.0;
c_size_t window_crossings = 0;
c_size_t cp1_precedes = 0;
c_size_t cp2_precedes = 0;
// 3. Deploy the Bounded Look-Ahead Sliding Scanner
// Loop through every single occurrence of cp1
for (c_size_t i = 0; i < len1; i++) {
c_size_t pos1 = o1[i];
// Find the first index in o2 where the element might be in range: pos2 >= pos1 - window_size
// We initialize look_idx using a basic optimization strategy, or start from 0
c_size_t look_idx = 0;
while (look_idx < len2) {
c_size_t pos2 = o2[look_idx];
// Condition A: pos2 is too far behind pos1, keep advancing the scan
if (pos2 < pos1 && (pos1 - pos2) > window_size) {
look_idx++;
continue;
}
// Condition B: pos2 is too far ahead of pos1, we can stop evaluating for this pos1
if (pos2 > pos1 && (pos2 - pos1) > window_size) {
break;
}
// Condition C: pos2 falls within the safe sliding context window boundary limit!
c_size_t dist = (pos1 > pos2) ? (pos1 - pos2) : (pos2 - pos1);
if (dist < min_dist) {
min_dist = dist;
}
total_dist_sum += (double)dist;
window_crossings++;
if (pos1 < pos2) {
cp1_precedes++; // cp1 appears BEFORE cp2
} else if (pos1 > pos2) {
cp2_precedes++; // cp2 appears BEFORE cp1
} else {
// Identity overlaps count as neutral crossings (rare for unique terms)
}
look_idx++;
}
}
// 4. Populate analytics result payloads accurately
if (window_crossings > 0) {
*out_min_dist = min_dist;
*out_avg_dist = total_dist_sum / (double)window_crossings;
*out_co_occurrences = window_crossings;
*out_cp1_precedes_count = cp1_precedes;
*out_cp2_precedes_count = cp2_precedes;
}
return C_ERR_OK;
}
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpCharGraph_ExportJSON(c_NlpCharGraph_t* self, const char* filepath) {
if (!self || !self->buckets || !filepath) return C_ERR_PARAM;
c_StringBuffer_t sb;
c_err_t err = c_StringBuffer_Init(&sb, 4096);
if (err != C_ERR_OK) return err;
err = c_StringBuffer_AppendStr(&sb, "{\n");
err |= c_StringBuffer_AppendStr(&sb, " \"meta\": {\n");
char meta_buf[128];
snprintf(meta_buf, sizeof(meta_buf), " \"total_words\": %lu,\n \"total_unique\": %lu\n",
(unsigned long)self->total_words, (unsigned long)self->total_unique);
err |= c_StringBuffer_AppendStr(&sb, meta_buf);
err |= c_StringBuffer_AppendStr(&sb, " },\n");
err |= c_StringBuffer_AppendStr(&sb, " \"nodes\": [\n");
if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
c_size_t nodes_serialized = 0;
// 外层遍历所有桶
for (c_size_t i = 0; i < self->bucket_count; i++) {
c_NlpCharNode_t* node = self->buckets[i];
// 核心修正：内层沿着冲突拉链链表深度追踪，把每一个被掩埋的字（如“模”）都全部挖出来导出
while (node != NULL) {
char utf8_word[8];
c_size_t written_len = 0;
if (c_utf8_from_unicode(node->codepoint, utf8_word, &written_len) != C_ERR_OK) {
node = node->next;
continue;
}
if (nodes_serialized > 0) {
err |= c_StringBuffer_AppendStr(&sb, ",\n");
}
err |= c_StringBuffer_AppendStr(&sb, " {\n");
char node_info[256];
if (utf8_word[0] == '"' || utf8_word[0] == '\\') {
snprintf(node_info, sizeof(node_info),
" \"char\": \"\\%s\",\n \"codepoint\": %u,\n \"frequency\": %lu,\n",
utf8_word, node->codepoint, (unsigned long)node->frequency);
} else {
snprintf(node_info, sizeof(node_info),
" \"char\": \"%s\",\n \"codepoint\": %u,\n \"frequency\": %lu,\n",
utf8_word, node->codepoint, (unsigned long)node->frequency);
}
err |= c_StringBuffer_AppendStr(&sb, node_info);
// 序列化 offsets
err |= c_StringBuffer_AppendStr(&sb, " \"offsets\": [");
for (c_size_t k = 0; k < node->offset_list.count; k++) {
char val_buf[32];
snprintf(val_buf, sizeof(val_buf), "%lu%s",
(unsigned long)node->offset_list.offsets[k],
(k + 1 < node->offset_list.count) ? ", " : "");
err |= c_StringBuffer_AppendStr(&sb, val_buf);
}
err |= c_StringBuffer_AppendStr(&sb, "],\n");
// 序列化 next_chars
err |= c_StringBuffer_AppendStr(&sb, " \"next_chars\": {");
c_bool_t edge_written = C_FALSE;
for (c_size_t bucket_idx = 0; bucket_idx < node->relation_capacity; bucket_idx++) {
c_NlpCharEdge_t* edge = node->next_chars[bucket_idx];
while (edge != NULL) {
char edge_utf8[8];
c_size_t e_len = 0;
if (c_utf8_from_unicode(edge->target_cp, edge_utf8, &e_len) == C_ERR_OK) {
char edge_buf[128];
const char* comma_prefix = edge_written ? ", " : "";
if (edge_utf8[0] == '"' || edge_utf8[0] == '\\') {
snprintf(edge_buf, sizeof(edge_buf), "%s\"\\%s\": %lu",
comma_prefix, edge_utf8, (unsigned long)edge->co_count);
} else {
snprintf(edge_buf, sizeof(edge_buf), "%s\"%s\": %lu",
comma_prefix, edge_utf8, (unsigned long)edge->co_count);
}
err |= c_StringBuffer_AppendStr(&sb, edge_buf);
edge_written = C_TRUE;
}
edge = edge->next;
}
}
err |= c_StringBuffer_AppendStr(&sb, "}\n");
err |= c_StringBuffer_AppendStr(&sb, " }");
nodes_serialized++;
// 顺着冲突拉链移动到下一个发生哈希冲突的字节点
node = node->next;
}
if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
}
err |= c_StringBuffer_AppendStr(&sb, "\n ]\n}\n");
if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
FILE* file = fopen(filepath, "wb");
if (!file) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
c_size_t expected_size = sb.size;
size_t written = fwrite(sb.buffer, 1, expected_size, file);
fclose(file);
c_StringBuffer_Destroy(&sb);
if (written != expected_size) return C_ERR_FAIL;
return C_ERR_OK;
}
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/**
* @brief Helper utility to safely extract value payloads trapped between specific JSON keys and brackets.
* Example: transforms ' "char": "大", ' into '大'
*/
static void c_Nlp_ExtractJsonValue(const char* line, const char* key, char* dest_val, size_t dest_max) {
dest_val[0] = '\0';
char* key_pos = strstr(line, key);
if (!key_pos) return;
char* start = strchr(key_pos + strlen(key), ':');
if (!start) return;
start++; // Step past ':'
// Skip white spaces and opening quotes
while (*start == ' ' || *start == '\t' || *start == '"') {
start++;
}
size_t idx = 0;
while (*start != '\0' && *start != '"' && *start != ',' && *start != '\n' && *start != '\r' && *start != ']' && idx < dest_max - 1) {
// Handle escaped JSON string properties
if (*start == '\\' && *(start + 1) != '\0') {
start++;
}
dest_val[idx++] = *start++;
}
dest_val[idx] = '\0';
// Trim trailing tracking spaces if any present
while (idx > 0 && (dest_val[idx - 1] == ' ' || dest_val[idx - 1] == '\t' || dest_val[idx - 1] == '}')) {
dest_val[--idx] = '\0';
}
}
c_err_t c_NlpCharGraph_ImportJSON(c_NlpCharGraph_t* self, const char* filepath, c_size_t bucket_count) {
if (!self || !filepath || bucket_count == 0) return C_ERR_PARAM;
FILE* file = fopen(filepath, "rb");
if (!file) return C_ERR_FAIL;
// Detect and safely strip standard UTF-8 BOM if present
unsigned char check_bom[3];
if (fread(check_bom, 1, 3, file) == 3 && check_bom[0] == 0xEF && check_bom[1] == 0xBB && check_bom[2] == 0xBF) {
// BOM skipped successfully
} else {
fseek(file, 0, SEEK_SET);
}
c_StringBuffer_t line_sb;
c_err_t err = c_StringBuffer_Init(&line_sb, 512);
if (err != C_ERR_OK) {
fclose(file);
return err;
}
c_size_t total_words = 0;
c_size_t relation_cap = 16; // Default initialization relation table width
// Fixed Buffer Upgrade: Secure 4096-byte stack arrays to completely negate truncation risks
char val_buf[4096];
// Phase 1: Parse Global Metadata Attributes
while (c_utf8_file_readline(file, &line_sb) == C_ERR_OK) {
if (strstr(line_sb.buffer, "\"total_words\"") != NULL) {
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"total_words\"", val_buf, sizeof(val_buf));
total_words = (c_size_t)strtoul(val_buf, NULL, 10);
}
if (strstr(line_sb.buffer, "\"nodes\"") != NULL) {
break; // Metadata block parsed, moving onto array entries
}
}
// Phase 2: Structural Graph Memory Instantiation
err = c_NlpCharGraph_Init(self, bucket_count, relation_cap);
if (err != C_ERR_OK) {
c_StringBuffer_Destroy(&line_sb);
fclose(file);
return err;
}
self->total_words = total_words;
c_NlpCharNode_t* active_node = NULL;
// Phase 3: Sequential Structural Node Deserialization
while (c_utf8_file_readline(file, &line_sb) == C_ERR_OK) {
// Ignore standalone array boundaries markers
if (strstr(line_sb.buffer, "{") != NULL && strstr(line_sb.buffer, "\"char\"") == NULL) {
continue;
}
// Parse individual token parameters
if (strstr(line_sb.buffer, "\"codepoint\"") != NULL) {
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"codepoint\"", val_buf, sizeof(val_buf));
c_ucs4_t cp = (c_ucs4_t)strtoul(val_buf, NULL, 10);
// Reconstruct utilizing standard chaining allocator to secure the linked pointers list
active_node = c_NlpCharGraph_GetOrCreateNode(self, cp);
if (!active_node) {
c_NlpCharGraph_Destroy(self);
c_StringBuffer_Destroy(&line_sb);
fclose(file);
return C_ERR_NOMEM;
}
}
else if (strstr(line_sb.buffer, "\"frequency\"") != NULL && active_node) {
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"frequency\"", val_buf, sizeof(val_buf));
active_node->frequency = (c_size_t)strtoul(val_buf, NULL, 10);
}
else if (strstr(line_sb.buffer, "\"offsets\"") != NULL && active_node) {
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"offsets\"", val_buf, sizeof(val_buf));
// Safe continuous array chunk tokenization using reentrant strtok mechanics
char* save_ptr = NULL;
char* token = c_utf8_strtok(val_buf, ", ", &save_ptr);
while (token != NULL) {
if (*token != '\0') {
c_size_t offset_val = (c_size_t)strtoul(token, NULL, 10);
c_NlpOffsetList_Add(&active_node->offset_list, offset_val);
}
token = c_utf8_strtok(NULL, ", ", &save_ptr);
}
}
else if (strstr(line_sb.buffer, "\"next_chars\"") != NULL && active_node) {
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"next_chars\"", val_buf, sizeof(val_buf));
// Process directional map entries: e.g., '"I": 1, "型": 1'
char* save_ptr = NULL;
char* pair_token = c_utf8_strtok(val_buf, ", ", &save_ptr);
while (pair_token != NULL) {
char* colon = strchr(pair_token, ':');
if (colon) {
*colon = '\0';
char edge_char[4096]; // Secure localized tracking array width
strncpy(edge_char, pair_token, sizeof(edge_char) - 1);
edge_char[sizeof(edge_char) - 1] = '\0';
// Strip enclosing quote strings out of target boundaries
char* edge_clean = edge_char;
while (*edge_clean == ' ' || *edge_clean == '\t' || *edge_clean == '"') {
edge_clean++;
}
size_t elen = strlen(edge_clean);
while (elen > 0 && (edge_clean[elen - 1] == '"' || edge_clean[elen - 1] == ' ' || edge_clean[elen - 1] == '\t')) {
edge_clean[--elen] = '\0';
}
if (elen > 0) {
c_ucs4_t target_cp = 0;
c_size_t bytes_step = 0;
// Transform character text string representation straight back into standard 32-bit integer points
if (c_utf8_to_unicode(edge_clean, &target_cp, &bytes_step) == C_ERR_OK) {
c_size_t edge_freq = (c_size_t)strtoul(colon + 1, NULL, 10);
// Rehydrate co-occurrence link matrices
for (c_size_t f = 0; f < edge_freq; f++) {
c_NlpCharNode_AddRelation(active_node->next_chars, active_node->relation_capacity, target_cp);
}
}
}
}
pair_token = c_utf8_strtok(NULL, ", ", &save_ptr);
}
}
}
c_StringBuffer_Destroy(&line_sb);
fclose(file);
return C_ERR_OK;
}
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpCharGraph_PredictNext(c_NlpCharGraph_t* self, c_ucs4_t current_char, c_NlpPredictMode_t mode,
c_ucs4_t* out_next_char, double* out_confidence) {
if (!self || !out_next_char || !out_confidence) {
return C_ERR_PARAM;
}
// 1. 在主哈希拉链表中定位当前字的节点
c_NlpCharNode_t* node = c_NlpCharGraph_GetNode(self, current_char);
if (!node || node->frequency == 0) {
return C_ERR_FAIL; // 图中不存在这个字，或者该字从未作为主词出现过
}
// 2. 遍历该节点的所有 next_chars 边，统计后续字的总权重和最大频次
c_size_t total_co_count = 0;
c_size_t max_co_count = 0;
c_ucs4_t best_target_cp = 0;
for (c_size_t i = 0; i < node->relation_capacity; i++) {
c_NlpCharEdge_t* edge = node->next_chars[i];
while (edge != NULL) {
total_co_count += edge->co_count;
// 贪婪模式所需的条件：记录最大频次
if (edge->co_count > max_co_count) {
max_co_count = edge->co_count;
best_target_cp = edge->target_cp;
}
edge = edge->next;
}
}
// 如果该字没有任何后续关联（比如它是文章的最后一个字且仅出现了一次）
if (total_co_count == 0) {
return C_ERR_FAIL;
}
// 3. 根据预测模式确定输出
if (mode == C_PREDICT_MODE_GREEDY) {
// 【贪婪模式】：直接选择后序关联最强、出现次数最多的字
*out_next_char = best_target_cp;
*out_confidence = (double)max_co_count / (double)total_co_count;
}
else if (mode == C_PREDICT_MODE_SAMPLING) {
// 【随机抽样模式】：基于一阶马尔可夫链的条件概率分布进行轮盘赌抽样
// 生成一个 0.0 到 1.0 之间的随机数
double random_pivot = (double)rand() / (double)RAND_MAX;
double current_accumulator = 0.0;
c_bool_t sampled = C_FALSE;
for (c_size_t i = 0; i < node->relation_capacity; i++) {
c_NlpCharEdge_t* edge = node->next_chars[i];
while (edge != NULL) {
double probability = (double)edge->co_count / (double)total_co_count;
current_accumulator += probability;
// 落在当前随机区间内，命中该字符
if (random_pivot <= current_accumulator) {
*out_next_char = edge->target_cp;
*out_confidence = probability;
sampled = C_TRUE;
break;
}
edge = edge->next;
}
if (sampled) break;
}
// 兜底保护，防止浮点数精度截断导致未命中
if (!sampled) {
*out_next_char = best_target_cp;
*out_confidence = (double)max_co_count / (double)total_co_count;
}
} else {
return C_ERR_PARAM;
}
return C_ERR_OK;
}
cNLP/Library/c_NlpCharGraph.h
+116
View File
@@ -0,0 +1,116 @@
#ifndef INCLUDED_C_NLPCHARGRAPH_H
#define INCLUDED_C_NLPCHARGRAPH_H
#ifndef INCLUDED_C_BASE_H
#include <c_Base.h>
#endif /*INCLUDED_C_BASE_H*/
#ifndef INCLUDED_C_UTF8_H
#include <c_utf8.h>
#endif /*INCLUDED_C_UTF8_H*/
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
typedef enum {
C_PREDICT_MODE_GREEDY = 0, /* 贪婪模式：总是选择频次最高的最优解 */
C_PREDICT_MODE_SAMPLING = 1 /* 抽样模式：根据概率分布进行随机选择（更具多样性） */
} c_NlpPredictMode_t;
/* --- 1. 动态位置数组（记录某个字在文章中出现的所有索引位置） --- */
typedef struct {
c_size_t* offsets; // 存储绝对位置的数组（如：第0, 15, 234个字）
c_size_t count; // 出现频次 (次数)
c_size_t capacity; // 动态数组容量
} c_NlpOffsetList_t;
/* --- 2. 邻接边结构体（记录当前字与前后字的关系及共现次数） --- */
typedef struct c_NlpCharEdge_t {
c_ucs4_t target_cp; // 关联字的 Unicode 码点
c_size_t co_count; // 在邻接上下文中共同出现的次数
struct c_NlpCharEdge_t* next; // 冲突链表指针（使用哈希拉链法或链表）
} c_NlpCharEdge_t;
/* --- 3. 核心：Unicode 字符语义拓扑节点 --- */
typedef struct c_NlpCharNode_t{
c_ucs4_t codepoint;
c_size_t frequency;
c_NlpOffsetList_t offset_list;
// Refactored pointers managing arbitrarily sized hash bucket tables
c_NlpCharEdge_t** next_chars; // Forward Bigram Hash Buckets array
c_NlpCharEdge_t** prev_chars; // Backward Bigram Hash Buckets array
c_size_t relation_capacity; // Current allocated size for next_chars and prev_chars
struct c_NlpCharNode_t* next;
} c_NlpCharNode_t;
typedef struct {
c_NlpCharNode_t** buckets; // 主哈希表：存储文章中所有出现过的唯一字节点
c_size_t bucket_count; // 主哈希桶大小（如 1024 或 4096，取决于去重后的字数）
c_size_t relation_default_capacity; // The default bucket capacity configured for sub-nodes
c_size_t total_unique; // 文章中不重复的汉字总量
c_size_t total_words; // 整篇文章的总字数（包括标点）
} c_NlpCharGraph_t;
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpCharGraph_Init(c_NlpCharGraph_t* self, c_size_t bucket_count, c_size_t relation_default_capacity);
c_NlpCharNode_t* c_NlpCharGraph_GetNode(c_NlpCharGraph_t* self, c_ucs4_t cp);
c_err_t c_NlpCharGraph_Process(c_NlpCharGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length);
void c_NlpCharGraph_Destroy(c_NlpCharGraph_t* self);
/**
* @brief Analyzes structural proximity metrics between two tokens within a sliding context window.
* @param self Pointer to the active initialized graph context block.
* @param cp1 The reference source Unicode character point.
* @param cp2 The target matching Unicode character point to measure.
* @param window_size Maximum token span distance threshold allowed (e.g., 5 tokens).
* @param out_min_dist Destination tracking pointer for the absolute minimum token distance.
* @param out_avg_dist Destination tracking pointer for the calculated average token distance.
* @param out_co_occurrences Destination tracking pointer counting total window crossing events.
* @param out_cp1_precedes_count Tracks how many times cp1 appeared BEFORE cp2 within the window.
* @param out_cp2_precedes_count Tracks how many times cp2 appeared BEFORE cp1 within the window.
* @return c_err_t C_ERR_OK on successful analytical completion, or C_ERR_PARAM if arguments are invalid.
*/
c_err_t c_NlpCharGraph_AnalyzeDistanceWindow(c_NlpCharGraph_t* self, c_ucs4_t cp1, c_ucs4_t cp2, c_size_t window_size,
c_size_t* out_min_dist, double* out_avg_dist, c_size_t* out_co_occurrences,
c_size_t* out_cp1_precedes_count, c_size_t* out_cp2_precedes_count);
/**
* @brief Exports the entire Unicode semantics graph structure to a file in valid JSON format.
* Automatically translates 32-bit codepoints back to human-readable UTF-8 strings.
* @param self Pointer to the active processed graph context block.
* @param filepath Destination path on disk where the JSON file will be saved.
* @return c_err_t C_ERR_OK on complete success, C_ERR_PARAM on invalid parameters, or C_ERR_FAIL on file access errors.
*/
c_err_t c_NlpCharGraph_ExportJSON(c_NlpCharGraph_t* self, const char* filepath);
/**
* @brief Parses an exported JSON text file and reconstructs the full Unicode graph structure in memory.
* @param self Pointer to an uninitialized or empty c_NlpCharGraph_t instance.
* @param filepath Path to the source JSON file on disk.
* @param bucket_count Main hash bucket size to use for the new graph context.
* @return c_err_t C_ERR_OK on complete success, C_ERR_PARAM on invalid inputs, or C_ERR_FAIL on structural parsing failures.
*/
c_err_t c_NlpCharGraph_ImportJSON(c_NlpCharGraph_t* self, const char* filepath, c_size_t bucket_count);
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/**
* @brief 根据当前已出现的字，预测并确定后续最应该输出什么字
* @param self 指向已完成语料分析的拓扑图实例指针
* @param current_char 当前最新出现的字（Unicode 码点）
* @param mode 选择的预测模式（贪婪 或 抽样）
* @param out_next_char 接收预测出的下一个字的指针
* @param out_confidence 接收该预测字的置信度（概率值，0.0 ~ 1.0）
* @return c_err_t C_ERR_OK 表示成功预测，C_ERR_PARAM 表示参数错误，C_ERR_FAIL 表示该字是生僻词/无后续关联
*/
c_err_t c_NlpCharGraph_PredictNext(c_NlpCharGraph_t* self, c_ucs4_t current_char, c_NlpPredictMode_t mode,
c_ucs4_t* out_next_char, double* out_confidence);
#endif /*INCLUDED_C_NLPCHARGRAPH_H*/
cNLP/Library/c_NlpCharGraph.t.c
+189
View File
@@ -0,0 +1,189 @@
#include "c_NlpCharGraph.h"
#include <stdlib.h>
#include <stdio.h>
#include "c_StringBuffer.h"
#include "c_utf8_file.h"
#define RUN_TEST(test_case, name) \
do { \
printf("[RUN] %s... ", name); \
if (test_case) { \
printf("\033[32mPASSED\033[0m\n"); \
} else { \
printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \
return C_ERR_FAIL; \
} \
} while(0)
static c_err_t test(void) {
/* 29. Arbitrary-Width Character Relation Graph Testing */
c_NlpCharGraph_t var_graph;
// Initialize graph with 512 main buckets and an arbitrary custom relation layout size of 128 channels
c_err_t err = c_NlpCharGraph_Init(&var_graph, 512, 128);
RUN_TEST(err == C_ERR_OK, "Variable-width graph initialization returns C_ERR_OK");
c_ucs4_t sample_text[] = { 0x540C, 0x6B65, 0x540C }; // '同', '步', '同'
err = c_NlpCharGraph_Process(&var_graph, sample_text, 3);
RUN_TEST(err == C_ERR_OK && var_graph.total_unique == 2, "Processed text array inside flexible capacity container");
c_NlpCharNode_t* node_tong = c_NlpCharGraph_GetNode(&var_graph, 0x540C); // '同'
RUN_TEST(node_tong != NULL && node_tong->relation_capacity == 128, "Verified node dynamically inherited the arbitrary 128-bucket relationship size setting");
RUN_TEST(node_tong->next_chars != NULL && node_tong->prev_chars != NULL, "relational heap allocation arrays successfully bound");
c_NlpCharGraph_Destroy(&var_graph);
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/* 30. Corrected Context Window Proximity Analytical Verification */
c_NlpCharGraph_t win_graph;
c_NlpCharGraph_Init(&win_graph, 1024, C_UNICODE_MAX);
// Corpus structural token layout: "A B B C A" -> Test multi-point matching
// Let 0x0041 = 'A', 0x0042 = 'B'
c_ucs4_t cluster_corpus[] = { 0x0041, 0x0042, 0x0042, 0x0043, 0x0041 };
err = c_NlpCharGraph_Process(&win_graph, cluster_corpus, 5);
RUN_TEST(err == C_ERR_OK, "Cluster corpus parsed inside window context analyzer container");
c_size_t min_d = 0;
double avg_d = 0.0;
c_size_t cross_count = 0;
c_size_t count_precedes_1 = 0;
c_size_t count_precedes_2 = 0;
// Analyze token proximity between 'A' (Index 0, 4) and 'B' (Index 1, 2) with a window threshold size of 3
// Target matches:
// For A at 0: B at 1 (dist 1, A precedes), B at 2 (dist 2, A precedes) -> 2 pairs
// For A at 4: B at 1 (dist 3, B precedes), B at 2 (dist 2, B precedes) -> 2 pairs
// Total co-occurrences expected = 4 crossings. Min distance = 1.
// Total distance sum = 1 + 2 + 3 + 2 = 8. Avg distance = 8 / 4 = 2.0.
err = c_NlpCharGraph_AnalyzeDistanceWindow(&win_graph, 0x0041, 0x0042, 3,
&min_d, &avg_d, &cross_count,
&count_precedes_1, &count_precedes_2);
RUN_TEST(err == C_ERR_OK, "Corrected window distance execution completed successfully");
RUN_TEST(cross_count == 4, "Accurately extracted exactly 4 window crossing combinations across the cluster");
RUN_TEST(min_d == 1, "Accurately isolated absolute minimum distance down to 1 token ('A' at 0 vs 'B' at 1)");
RUN_TEST(avg_d == 2.0, "Accurately evaluated average mathematical mean distance at 2.0 tokens precisely");
RUN_TEST(count_precedes_1 == 2, "Verified 'A' preceded 'B' exactly 2 times across window evaluation frames");
RUN_TEST(count_precedes_2 == 2, "Verified 'B' preceded 'A' exactly 2 times across window evaluation frames");
c_NlpCharGraph_Destroy(&win_graph);
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/* 31. Unicode Character Graph JSON Export Verification */
c_NlpCharGraph_t json_graph;
const char* output_json_path = "nlp_graph_export.json";
// Initialize graph context
err = c_NlpCharGraph_Init(&json_graph, 16, 4);
RUN_TEST(err == C_ERR_OK, "JSON Graph context initialized successfully");
// Input string: "AI大模型" -> Translated into explicit unicode codepoint stream arrays
c_ucs4_t export_corpus[] = { 0x0041, 0x0049, 0x5927, 0x6A21, 0x578B };
err = c_NlpCharGraph_Process(&json_graph, export_corpus, 5);
RUN_TEST(err == C_ERR_OK, "Export corpus strings parsed smoothly inside graph matrices");
// Execute JSON Export API pipeline
err = c_NlpCharGraph_ExportJSON(&json_graph, output_json_path);
RUN_TEST(err == C_ERR_OK, "c_NlpCharGraph_ExportJSON successfully generated JSON file on disk");
// Read file back to verify it contains valid structural characters
c_StringBuffer_t read_verify_sb;
c_StringBuffer_Init(&read_verify_sb, 1024);
err = c_utf8_file_read(output_json_path, &read_verify_sb);
RUN_TEST(err == C_ERR_OK, "Successfully read generated JSON output file back into workspace");
RUN_TEST(strstr(read_verify_sb.buffer, "\"total_words\": 5") != NULL, "JSON metadata section successfully parsed");
RUN_TEST(strstr(read_verify_sb.buffer, "\"char\": \"大\"") != NULL, "Unicode codepoint 0x5927 successfully serialized to UTF-8 character string '大'");
RUN_TEST(strstr(read_verify_sb.buffer, "\"next_chars\":") != NULL, "Directional adjacency edge transition attributes successfully embedded");
// Clean up temporary files and memory structures
c_StringBuffer_Destroy(&read_verify_sb);
c_NlpCharGraph_Destroy(&json_graph);
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/* 32. Unicode Character Graph JSON Import Validation */
c_NlpCharGraph_t imported_graph;
const char* target_json_src = "nlp_graph_export.json";
// Dependencies checkpoint: Ensure file exists before running verification passes
// (Generated by calling c_NlpCharGraph_ExportJSON in the previous step)
err = c_NlpCharGraph_ImportJSON(&imported_graph, target_json_src, 1024);
RUN_TEST(err == C_ERR_OK, "c_NlpCharGraph_ImportJSON parsed JSON text stream and built the graph successfully");
// Validate Structural Reconstitution Fidelity Metrics
RUN_TEST(imported_graph.total_words == 5, "Reconstructed metadata 'total_words' matches value: 5");
// Fetch and check nodes internally
c_NlpCharNode_t* check_da = c_NlpCharGraph_GetNode(&imported_graph, 0x5927); // '大'
RUN_TEST(check_da != NULL, "Unicode Node 0x5927 ('大') successfully verified in bucket storage");
if (check_da) {
RUN_TEST(check_da->frequency == 1, "Frequency counter holds correct historical state value: 1");
RUN_TEST(check_da->offset_list.count == 1, "Offsets dynamic list successfully loaded elements");
// Trace and check directional bigram linkages
c_size_t min_d = 0;
double avg_d = 0.0;
c_size_t crossings = 0;
c_size_t p1 = 0, p2 = 0;
err = c_NlpCharGraph_AnalyzeDistanceWindow(&imported_graph, 0x5927, 0x6A21, 2, &min_d, &avg_d, &crossings, &p1, &p2); // '大' vs '模'
RUN_TEST(err == C_ERR_OK && crossings == 1, "Relational bigram matching layers successfully operational post-import");
}
c_NlpCharGraph_Destroy(&imported_graph);
// remove(output_json_path); // Remove transient file from disk space
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/* 31. 语言模型后续字预测验证 (c_NlpCharGraph_PredictNext) */
c_NlpCharGraph_t pred_graph;
c_NlpCharGraph_Init(&pred_graph, 1024, 16);
// 载入高重复、高确定性的测试语料："大模型大模型大项目"
// '大' 后面出现了 2 次 '模'，1 次 '项'。总后续数 = 3
c_ucs4_t train_data[] = {
0x5927, 0x6A21, 0x578B, // 大模型
0x5927, 0x6A21, 0x578B, // 大模型
0x5927, 0x9879, 0x76EE // 大项目
};
err = c_NlpCharGraph_Process(&pred_graph, train_data, 9);
RUN_TEST(err == C_ERR_OK, "Prediction training data parsed smoothly");
c_ucs4_t predicted_char = 0;
double confidence = 0.0;
// 进行确定性的贪婪模式预测：输入 '大' (0x5927)
// 预期：'模' (0x6A21) 出现的概率是 2/3 = 66.67%，远高于 '项' 的 1/3
err = c_NlpCharGraph_PredictNext(&pred_graph, 0x5927, C_PREDICT_MODE_GREEDY, &predicted_char, &confidence);
RUN_TEST(err == C_ERR_OK, "PredictNext completed execution path successfully");
RUN_TEST(predicted_char == 0x6A21, "Greedy mode correctly deterministically chooses '模' following '大'");
// 校验置信度 2.0 / 3.0 近似等于 0.666667
RUN_TEST(confidence > 0.66 && confidence < 0.67, "Confidence value evaluated exactly at 66.67%%");
// 测试生僻词/无后续关联的边界安全
err = c_NlpCharGraph_PredictNext(&pred_graph, 0x76EE, C_PREDICT_MODE_GREEDY, &predicted_char, &confidence); // '目' 是文章末尾
RUN_TEST(err == C_ERR_FAIL, "PredictNext safely rejects character with zero trailing context links");
c_NlpCharGraph_Destroy(&pred_graph);
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
return C_ERR_OK;
}
int main(int argc, char** argv){
return test();
return 0;
}
cNLP/Library/c_NlpLanguageModel.c
+94
View File
@@ -0,0 +1,94 @@
#include <c_NlpLanguageModel.h>
#include <c_Memory.h>
c_err_t c_NlpLanguageModel_Init(c_NlpLanguageModel_t* self, c_size_t max_n, c_size_t bucket_count) {
if (!self || max_n < 2 || bucket_count == 0) return C_ERR_PARAM;
self->max_n = max_n;
self->backoff_alpha = 0.4; // Industrial standard alpha decay baseline configuration
// Allocate space for all N-gram tiers down to 2-gram (size = max_n - 1)
c_size_t graph_count = max_n - 1;
self->levels = (c_NlpNgramGraph_t*)C_ALLOC(sizeof(c_NlpNgramGraph_t) * graph_count);
if (!self->levels) return C_ERR_NOMEM;
// Initialize sub-graphs from highest order down to lowest (Bigram)
for (c_size_t i = 0; i < graph_count; i++) {
c_size_t current_n = max_n - i;
c_err_t err = c_NlpNgramGraph_Init(&self->levels[i], current_n, bucket_count, 16);
if (err != C_ERR_OK) {
// Rollback previously initialized sub-graphs on allocation failure
for (c_size_t k = 0; k < i; k++) {
c_NlpNgramGraph_Destroy(&self->levels[k]);
}
C_FREE(self->levels);
self->levels = NULL;
return err;
}
}
return C_ERR_OK;
}
c_err_t c_NlpLanguageModel_Train(c_NlpLanguageModel_t* self, const c_ucs4_t* unicode_array, c_size_t length) {
if (!self || !self->levels || !unicode_array || length == 0) return C_ERR_PARAM;
c_size_t graph_count = self->max_n - 1;
// Feed data to train all nested sub-graphs simultaneously
for (c_size_t i = 0; i < graph_count; i++) {
c_err_t err = c_NlpNgramGraph_Process(&self->levels[i], unicode_array, length);
if (err != C_ERR_OK) return err;
}
return C_ERR_OK;
}
c_err_t c_NlpLanguageModel_PredictSmooth(c_NlpLanguageModel_t* self, const c_ucs4_t* history_context, c_size_t history_len,
c_ucs4_t* out_next_char, double* out_confidence) {
if (!self || !self->levels || !history_context || history_len == 0 || !out_next_char || !out_confidence) {
return C_ERR_PARAM;
}
c_size_t graph_count = self->max_n - 1;
double current_alpha_multiplier = 1.0;
// Step down from the highest-order N-gram down to the baseline Bigram level
for (c_size_t i = 0; i < graph_count; i++) {
c_NlpNgramGraph_t* current_graph = &self->levels[i];
c_size_t expected_history_len = current_graph->n_value - 1;
// If our current available history sequence matches the tier's required context length
if (history_len >= expected_history_len) {
// Extract a sliced pointer matching exactly the trailing edge suffix of our history
c_size_t offset_start = history_len - expected_history_len;
const c_ucs4_t* sliced_history = &history_context[offset_start];
double baseline_confidence = 0.0;
// Attempt a retrieval match execution path
c_err_t err = c_NlpNgramGraph_PredictNext(current_graph, sliced_history, out_next_char, &baseline_confidence);
if (err == C_ERR_OK) {
// High-order match successful! Deduct confidence with our decay penalty multipliers
*out_confidence = baseline_confidence * current_alpha_multiplier;
return C_ERR_OK;
}
}
// If the query falls through (C_ERR_FAIL), cascade lower and apply the alpha smoothing penalty
current_alpha_multiplier *= self->backoff_alpha;
}
return C_ERR_FAIL; // Completely out of vocabulary context indicators across all internal graphs
}
void c_NlpLanguageModel_Destroy(c_NlpLanguageModel_t* self) {
if (!self || !self->levels) return;
c_size_t graph_count = self->max_n - 1;
for (c_size_t i = 0; i < graph_count; i++) {
c_NlpNgramGraph_Destroy(&self->levels[i]);
}
C_FREE(self->levels);
self->levels = NULL;
self->max_n = 0;
}
cNLP/Library/c_NlpLanguageModel.h
+38
View File
@@ -0,0 +1,38 @@
#ifndef INCLUDED_C_NLPLANGUAGEMODEL_H
#define INCLUDED_C_NLPLANGUAGEMODEL_H
#ifndef INCLUDED_C_NLPNGRAMGRAPH_H
#include <c_NlpNgramGraph.h>
#endif /*INCLUDED_C_NLPNGRAMGRAPH_H*/
typedef struct {
c_NlpNgramGraph_t* levels; /* Array of N-gram graphs sorted from highest order down to 2-gram */
c_size_t max_n; /* The maximum configuration order N of the model (e.g., 3 for Trigram) */
double backoff_alpha; /* Proportional smoothing decay factor (typically 0.4) used per back-off step */
} c_NlpLanguageModel_t;
/**
* @brief Initializes a multi-tier cascaded Language Model environment supporting back-off smoothing.
* @param self Pointer to the uninitialized language model structure.
* @param max_n Maximum model order threshold (e.g., 3 creates a Trigram + Bigram cascading mesh).
* @param bucket_count Main hash allocation array size mapped down to sub-graphs.
* @return c_err_t C_ERR_OK on successful initialization, or dynamic memory OOM codes.
*/
c_err_t c_NlpLanguageModel_Init(c_NlpLanguageModel_t* self, c_size_t max_n, c_size_t bucket_count);
/**
* @brief Core Prediction: Evaluates histories and executes automated back-off smoothing cascades.
*/
c_err_t c_NlpLanguageModel_PredictSmooth(c_NlpLanguageModel_t* self, const c_ucs4_t* history_context, c_size_t history_len,
c_ucs4_t* out_next_char, double* out_confidence);
/**
* @brief Destroys and flushes all cascaded hierarchy sub-graphs completely.
*/
void c_NlpLanguageModel_Destroy(c_NlpLanguageModel_t* self);
c_err_t c_NlpLanguageModel_Train(c_NlpLanguageModel_t* self, const c_ucs4_t* unicode_array, c_size_t length);
#endif /*INCLUDED_C_NLPLANGUAGEMODEL_H*/
cNLP/Library/c_NlpLanguageModel.t.c
+58
View File
@@ -0,0 +1,58 @@
#include "c_NlpLanguageModel.h"
#include <stdlib.h>
#include <stdio.h>
#define RUN_TEST(test_case, name) \
do { \
printf("[RUN] %s... ", name); \
if (test_case) { \
printf("\033[32mPASSED\033[0m\n"); \
} else { \
printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \
return C_ERR_FAIL; \
} \
} while(0)
static c_err_t test(void) {
/* 34. Katz Back-off Smoothing Cascade Verification */
c_NlpLanguageModel_t lm;
// Instantiate a 3-tier system (Trigram + Bigram cascaded backend framework mesh)
c_err_t err = c_NlpLanguageModel_Init(&lm, 3, 512);
RUN_TEST(err == C_ERR_OK, "Cascaded Language Model successfully initialized");
// Training set containing: "大模型" and "微模型"
c_ucs4_t text_stream[] = {
0x5927, 0x6A21, 0x578B, /* 大模型 */
0x5FAE, 0x6A21, 0x578B /* 微模型 */
};
err = c_NlpLanguageModel_Train(&lm, text_stream, 6);
RUN_TEST(err == C_ERR_OK, "Cascaded models successfully ingested corpus dataset streams");
c_ucs4_t next_prediction = 0;
double confidence_score = 0.0;
// --- Scenario A: Perfect high-order match ---
// Query context: "大模" -> Expected result: "型" (100% confidence from Trigram tier)
c_ucs4_t perfect_history[] = { 0x5927, 0x6A21 };
err = c_NlpLanguageModel_PredictSmooth(&lm, perfect_history, 2, &next_prediction, &confidence_score);
RUN_TEST(err == C_ERR_OK && next_prediction == 0x578B, "Scenario A: High-order match correctly resolves to '型'");
RUN_TEST(confidence_score == 1.0, "Scenario A: Direct hit returns full unpenalized confidence metrics");
// --- Scenario B: Unseen Trigram history forces a Back-off step ---
// Query context: "新模" -> The 3-gram "新模" is completely unseen in our training text
// Expected behavior: Trigram lookup fails. Model smoothly backs off to the Bigram tier matching only "模" -> "型".
// Mathematical projection: 1.0 baseline confidence * 0.4 alpha penalty = 0.4 adjusted score.
c_ucs4_t unseen_trigram_history[] = { 0x65B0, 0x6A21 }; // "新模"
err = c_NlpLanguageModel_PredictSmooth(&lm, unseen_trigram_history, 2, &next_prediction, &confidence_score);
RUN_TEST(err == C_ERR_OK, "Scenario B: Back-off routine successfully processes the unseen 3-gram phrase");
RUN_TEST(next_prediction == 0x578B, "Scenario B: Lower-order bigram successfully resolves '模' ➔ '型'");
RUN_TEST(confidence_score == 0.4, "Scenario B: Confidence safely records the 0.4 alpha smoothing decay penalty");
c_NlpLanguageModel_Destroy(&lm);
return C_ERR_OK;
}
int main(int argc, char** argv){
return test();
}
cNLP/Library/c_NlpNgramGraph.c
+242
View File
@@ -0,0 +1,242 @@
#include <c_NlpNgramGraph.h>
#include <c_Memory.h>
#include <stdlib.h>
#include <string.h>
/* FNV-1a 32位 滾動哈希，用於對變長 Unicode 陣列計算主桶索引 */
C_STATIC_FORCE_INLINE
c_size_t c_NlpNgram_HashContext(const c_ucs4_t* context, c_size_t len, c_size_t bucket_count) {
unsigned int hash = 2166136261U;
for (c_size_t i = 0; i < len; i++) {
hash ^= context[i];
hash *= 16777619U;
}
return (c_size_t)(hash % bucket_count);
}
/* 比較兩個上下文陣列是否完全一致 */
C_STATIC_FORCE_INLINE
c_bool_t c_NlpNgram_ContextEquals(const c_ucs4_t* ctx1, const c_ucs4_t* ctx2, c_size_t len) {
for (c_size_t i = 0; i < len; i++) {
if (ctx1[i] != ctx2[i]) return C_FALSE;
}
return C_TRUE;
}
/* 建立並初始化一個全新的歷史狀態節點 */
C_STATIC_FORCE_INLINE
c_NgramNode_t* c_NgramNode_Create(const c_ucs4_t* context, c_size_t ctx_len, c_size_t rel_cap) {
c_NgramNode_t* node = (c_NgramNode_t*)C_ALLOC(sizeof(c_NgramNode_t));
if (!node) return NULL;
node->frequency = 0;
node->relation_capacity = rel_cap;
node->next = NULL;
// 深拷貝上下文狀態陣列
node->context = (c_ucs4_t*)C_ALLOC(sizeof(c_ucs4_t) * ctx_len);
node->next_chars = (c_NgramEdge_t**)C_ALLOC(sizeof(c_NgramEdge_t*) * rel_cap);
if (!node->context || !node->next_chars) {
C_FREE(node->context);
C_FREE(node->next_chars);
C_FREE(node);
return NULL;
}
memcpy(node->context, context, sizeof(c_ucs4_t) * ctx_len);
for (c_size_t i = 0; i < rel_cap; i++) {
node->next_chars[i] = NULL;
}
return node;
}
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpNgramGraph_Init(c_NlpNgramGraph_t* self, c_size_t n_value, c_size_t bucket_count, c_size_t relation_default_capacity) {
if (!self || n_value < 2 || bucket_count == 0 || relation_default_capacity == 0) return C_ERR_PARAM;
self->n_value = n_value;
self->bucket_count = bucket_count;
self->relation_default_capacity = relation_default_capacity;
self->total_states = 0;
self->buckets = (c_NgramNode_t**)C_ALLOC(sizeof(c_NgramNode_t*) * bucket_count);
if (!self->buckets) return C_ERR_NOMEM;
for (c_size_t i = 0; i < bucket_count; i++) {
self->buckets[i] = NULL;
}
return C_ERR_OK;
}
/* 拉鏈法安全獲取或建立上下文狀態節點 */
static c_NgramNode_t* c_NlpNgramGraph_GetOrCreate(c_NlpNgramGraph_t* self, const c_ucs4_t* context) {
c_size_t ctx_len = self->n_value - 1;
c_size_t idx = c_NlpNgram_HashContext(context, ctx_len, self->bucket_count);
c_NgramNode_t* curr = self->buckets[idx];
c_NgramNode_t* prev = NULL;
while (curr != NULL) {
if (c_NlpNgram_ContextEquals(curr->context, context, ctx_len)) {
return curr; // 命中已有歷史狀態
}
prev = curr;
curr = curr->next;
}
// 未命中，建立新歷史狀態節點
c_NgramNode_t* new_node = c_NgramNode_Create(context, ctx_len, self->relation_default_capacity);
if (!new_node) return NULL;
if (prev == NULL) {
self->buckets[idx] = new_node;
} else {
prev->next = new_node;
}
self->total_states++;
return new_node;
}
/* 核心注入：滑動視窗掃描整篇文章語料 */
c_err_t c_NlpNgramGraph_Process(c_NlpNgramGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length) {
if (!self || !self->buckets || !unicode_array || length == 0) return C_ERR_PARAM;
c_size_t n = self->n_value;
// 如果文章字數小於 N 元模型所需的長度，無法建立狀態
if (length < n) return C_ERR_OK;
// 滑動視窗上界為 length - n + 1
for (c_size_t i = 0; i <= length - n; i++) {
// 1. 當前滑動視窗的起點指標，指向長度為 N-1 的歷史上下文
const c_ucs4_t* current_context = &unicode_array[i];
// 2. 獲取或建立對應的多元歷史節點
c_NgramNode_t* node = c_NlpNgramGraph_GetOrCreate(self, current_context);
if (!node) return C_ERR_NOMEM;
node->frequency++;
// 3. 提取第 N 個字作為預測轉移目標
c_ucs4_t target_char = unicode_array[i + n - 1];
// 4. 將預測目標掛入該歷史節點的 next_chars 邊哈希拉鏈中
c_size_t edge_idx = (c_size_t)(target_char % node->relation_capacity);
c_NgramEdge_t* edge = node->next_chars[edge_idx];
c_bool_t edge_found = C_FALSE;
while (edge != NULL) {
if (edge->target_cp == target_char) {
edge->co_count++;
edge_found = C_TRUE;
break;
}
edge = edge->next;
}
if (!edge_found) {
c_NgramEdge_t* new_edge = (c_NgramEdge_t*)C_ALLOC(sizeof(c_NgramEdge_t));
if (!new_edge) return C_ERR_NOMEM;
new_edge->target_cp = target_char;
new_edge->co_count = 1;
new_edge->next = node->next_chars[edge_idx];
node->next_chars[edge_idx] = new_edge;
}
}
return C_ERR_OK;
}
void c_NlpNgramGraph_Destroy(c_NlpNgramGraph_t* self) {
if (!self || !self->buckets || self->bucket_count == 0) return;
c_NgramNode_t** local_buckets = self->buckets;
c_size_t local_count = self->bucket_count;
// 搶先阻斷外部二次訪問漏洞
self->buckets = NULL;
self->bucket_count = 0;
self->total_states = 0;
for (c_size_t i = 0; i < local_count; i++) {
c_NgramNode_t* node = local_buckets[i];
while (node != NULL) {
c_NgramNode_t* next_node = node->next;
// 1. 釋放深層歷史上下文陣列
if (node->context != NULL) {
C_FREE(node->context);
}
// 2. 釋放後續預測鄰接邊拉鏈
if (node->next_chars != NULL) {
for (c_size_t k = 0; k < node->relation_capacity; k++) {
c_NgramEdge_t* edge = node->next_chars[k];
while (edge != NULL) {
c_NgramEdge_t* temp_edge = edge->next;
C_FREE(edge);
edge = temp_edge;
}
}
C_FREE(node->next_chars);
}
// 3. 釋放狀態節點本身
C_FREE(node);
node = next_node;
}
}
C_FREE(local_buckets);
}
c_err_t c_NlpNgramGraph_PredictNext(c_NlpNgramGraph_t* self, const c_ucs4_t* history_context,
c_ucs4_t* out_next_char, double* out_confidence) {
if (!self || !self->buckets || !history_context || !out_next_char || !out_confidence) {
return C_ERR_PARAM;
}
c_size_t ctx_len = self->n_value - 1;
c_size_t idx = c_NlpNgram_HashContext(history_context, ctx_len, self->bucket_count);
// 1. 定位多元歷史狀態
c_NgramNode_t* node = self->buckets[idx];
while (node != NULL) {
if (c_NlpNgram_ContextEquals(node->context, history_context, ctx_len)) {
break;
}
node = node->next;
}
// 如果語料庫中從未出現過這段連續的歷史短语（未命中）
if (!node || node->frequency == 0) {
return C_ERR_FAIL;
}
// 2. 統計該狀態下的邊權重，找出概率最高的轉移字符
c_size_t total_co_count = 0;
c_size_t max_co_count = 0;
c_ucs4_t best_target_cp = 0;
for (c_size_t i = 0; i < node->relation_capacity; i++) {
c_NgramEdge_t* edge = node->next_chars[i];
while (edge != NULL) {
total_co_count += edge->co_count;
if (edge->co_count > max_co_count) {
max_co_count = edge->co_count;
best_target_cp = edge->target_cp;
}
edge = edge->next;
}
}
if (total_co_count == 0) return C_ERR_FAIL;
// 输出預測最優解與馬爾可夫最大似然概率
*out_next_char = best_target_cp;
*out_confidence = (double)max_co_count / (double)total_co_count;
return C_ERR_OK;
}
cNLP/Library/c_NlpNgramGraph.h
+51
View File
@@ -0,0 +1,51 @@
#ifndef INCLUDED_C_NLPNGRAMGRAPH_H
#define INCLUDED_C_NLPNGRAMGRAPH_H
#ifndef INCLUDED_C_UTF8_H
#include <c_utf8.h>
#endif /*INCLUDED_C_UTF8_H*/
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
/* --- 1. N-gram 預測邊（記錄後續可能出現的單個字及頻次） --- */
typedef struct c_NgramEdge_t {
c_ucs4_t target_cp; /* 後續預測出的第 N 個字 */
c_size_t co_count; /* 該字在當前上下文後出現的次數 */
struct c_NgramEdge_t* next; /* 鄰接邊哈希衝突拉鏈 */
} c_NgramEdge_t;
/* --- 2. N-gram 上下文節點（代表長度為 N-1 的歷史狀態） --- */
typedef struct c_NgramNode_t {
c_ucs4_t* context; /* 歷史上下文陣列，長度為 N-1 */
c_size_t frequency; /* 該上下文總出現次數 */
c_NgramEdge_t** next_chars; /* 後續字哈希桶（動態分配） */
c_size_t relation_capacity; /* 邊哈希桶容量 */
struct c_NgramNode_t* next; /* 主哈希表衝突拉鏈指针（解決主桶碰撞） */
} c_NgramNode_t;
/* --- 3. 多元馬爾可夫鏈模型主控結構体 --- */
typedef struct {
c_NgramNode_t** buckets; /* 主哈希表（儲存所有不重複的歷史上下文狀態） */
c_size_t bucket_count; /* 主哈希桶大小 */
c_size_t n_value; /* 模型階數 N (例如：3 代表 Trigram 模型) */
c_size_t relation_default_capacity; /* 子節點邊哈希桶默認容量 */
c_size_t total_states; /* 記錄圖中唯一存在的歷史狀態總數 */
} c_NlpNgramGraph_t;
/* ------------------------------------------------------------------------------------------------------------------ */
/* */
c_err_t c_NlpNgramGraph_Init(c_NlpNgramGraph_t* self, c_size_t n_value, c_size_t bucket_count, c_size_t relation_default_capacity);
void c_NlpNgramGraph_Destroy(c_NlpNgramGraph_t* self);
c_err_t c_NlpNgramGraph_Process(c_NlpNgramGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length);
c_err_t c_NlpNgramGraph_PredictNext(c_NlpNgramGraph_t* self, const c_ucs4_t* history_context,
c_ucs4_t* out_next_char, double* out_confidence);
#endif /*INCLUDED_C_NLPNGRAMGRAPH_H*/
cNLP/Library/c_NlpNgramGraph.t.c
+52
View File
@@ -0,0 +1,52 @@
#include "c_NlpNgramGraph.h"
#include <stdlib.h>
#include <stdio.h>
#define RUN_TEST(test_case, name) \
do { \
printf("[RUN] %s... ", name); \
if (test_case) { \
printf("\033[32mPASSED\033[0m\n"); \
} else { \
printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \
return C_ERR_FAIL; \
} \
} while(0)
static c_err_t test(void) {
/* 33. 多元馬爾可夫鏈（Trigram）核心通路驗證 */
c_NlpNgramGraph_t trigram;
// 建立 3元 模型：需要根據 2 个字的历史預測第 3 个字
c_err_t err = c_NlpNgramGraph_Init(&trigram, 3, 512, 8);
RUN_TEST(err == C_ERR_OK, "Trigram Graph initialized cleanly");
// 載入預測語料："自然语言、自然界、自然语言"
// 當已知歷史短语是 "自然" 时，"语" 出现了 2 次，"界" 出现了 1 次
c_ucs4_t corpus_stream[] = {
0x81EA, 0x7136, 0x8BED, 0x8A00, // 自然语言
0x81EA, 0x7136, 0x754C, // 自然界
0x81EA, 0x7136, 0x8BED, 0x8A00 // 自然语言
};
err = c_NlpNgramGraph_Process(&trigram, corpus_stream, 11);
RUN_TEST(err == C_ERR_OK && trigram.total_states > 0, "Trigram processed training data stream");
// 構建查詢歷史：[0x81EA, 0x7136] ➔ 代表 "自然"
c_ucs4_t query_history[] = { 0x81EA, 0x7136 };
c_ucs4_t pred_res = 0;
double pred_conf = 0.0;
err = c_NlpNgramGraph_PredictNext(&trigram, query_history, &pred_res, &pred_conf);
RUN_TEST(err == C_ERR_OK, "PredictNext completed high-order context traversal paths");
// 預期：準確命中 "语" (0x8BED)，置信度為 2 / 3 = 66.67%
RUN_TEST(pred_res == 0x8BED, "Trigram language model predicts '语' given advanced context '自然'");
RUN_TEST(pred_conf > 0.66 && pred_conf < 0.67, "Context-bound model confidence targets 66.67%% correctly");
c_NlpNgramGraph_Destroy(&trigram);
return C_ERR_OK;
}
int main(int argc, char** argv){
return test();
}