pchen
e6a29bcdfd
c_NlpNgramGraph: 基于 n 个已经出现的词，预估后续可能出现的词 c_NlpLanguageModel: 如果有的词组合形式没有出现过，找到之前学习过最长类似的词句进行预测
705 lines
26 KiB
C
705 lines
26 KiB
C
#include <c_NlpCharGraph.h>
|
|
#include <c_Memory.h>
|
|
#include <stdio.h>
|
|
#include "c_StringBuffer.h"
|
|
#include "c_utf8_file.h"
|
|
|
|
|
|
C_STATIC_FORCE_INLINE
|
|
c_size_t c_NlpGraph_Hash(c_ucs4_t cp, c_size_t capacity) {
|
|
return (c_size_t)(cp % capacity);
|
|
}
|
|
|
|
C_STATIC_FORCE_INLINE
|
|
c_err_t c_NlpOffsetList_Init(c_NlpOffsetList_t* list) {
|
|
list->count = 0;
|
|
list->capacity = 4;
|
|
list->offsets = (c_size_t*)C_ALLOC(sizeof(c_size_t) * list->capacity);
|
|
return list->offsets ? C_ERR_OK : C_ERR_NOMEM;
|
|
}
|
|
|
|
static c_err_t c_NlpOffsetList_Add(c_NlpOffsetList_t* list, c_size_t offset) {
|
|
if (list->count >= list->capacity) {
|
|
c_size_t new_cap = list->capacity * 2;
|
|
c_size_t* new_offsets = (c_size_t*)C_ALLOC(sizeof(c_size_t) * new_cap);
|
|
if (!new_offsets) return C_ERR_NOMEM;
|
|
memcpy(new_offsets, list->offsets, sizeof(c_size_t) * list->count);
|
|
C_FREE(list->offsets);
|
|
list->offsets = new_offsets;
|
|
list->capacity = new_cap;
|
|
}
|
|
list->offsets[list->count++] = offset;
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
/* --- Dynamic Node Instantiation Engine --- */
|
|
static c_NlpCharNode_t* c_NlpCharNode_Create(c_ucs4_t cp, c_size_t relation_capacity) {
|
|
if (relation_capacity == 0) return NULL;
|
|
|
|
c_NlpCharNode_t* node = (c_NlpCharNode_t*)C_ALLOC(sizeof(c_NlpCharNode_t));
|
|
if (!node) return NULL;
|
|
|
|
node->codepoint = cp;
|
|
node->frequency = 0;
|
|
node->relation_capacity = relation_capacity;
|
|
|
|
if (c_NlpOffsetList_Init(&node->offset_list) != C_ERR_OK) {
|
|
C_FREE(node);
|
|
return NULL;
|
|
}
|
|
|
|
// Allocate variable-width hash array tables on the heap
|
|
node->next_chars = (c_NlpCharEdge_t**)C_ALLOC(sizeof(c_NlpCharEdge_t*) * relation_capacity);
|
|
node->prev_chars = (c_NlpCharEdge_t**)C_ALLOC(sizeof(c_NlpCharEdge_t*) * relation_capacity);
|
|
|
|
if (!node->next_chars || !node->prev_chars) {
|
|
C_FREE(node->next_chars);
|
|
C_FREE(node->prev_chars);
|
|
C_FREE(node->offset_list.offsets);
|
|
C_FREE(node);
|
|
return NULL;
|
|
}
|
|
|
|
// Zero out pointers across all newly configured custom slots
|
|
for (c_size_t i = 0; i < relation_capacity; i++) {
|
|
node->next_chars[i] = NULL;
|
|
node->prev_chars[i] = NULL;
|
|
}
|
|
|
|
return node;
|
|
}
|
|
|
|
static c_NlpCharNode_t* c_NlpCharGraph_GetOrCreateNode(c_NlpCharGraph_t* self, c_ucs4_t cp) {
|
|
c_size_t idx = c_NlpGraph_Hash(cp, self->bucket_count);
|
|
c_NlpCharNode_t* curr = self->buckets[idx];
|
|
c_NlpCharNode_t* prev = NULL;
|
|
|
|
// 沿拉链向下查找匹配的码点
|
|
while (curr != NULL) {
|
|
if (curr->codepoint == cp) {
|
|
return curr; // 完美命中已有节点
|
|
}
|
|
prev = curr;
|
|
curr = curr->next;
|
|
}
|
|
|
|
// 未命中冲突，创建全新节点并通过头插法/尾插法挂入拉链
|
|
c_NlpCharNode_t* new_node = c_NlpCharNode_Create(cp, self->relation_default_capacity);
|
|
if (!new_node) return NULL;
|
|
new_node->next = NULL;
|
|
|
|
if (prev == NULL) {
|
|
self->buckets[idx] = new_node; // 桶内第一个节点
|
|
} else {
|
|
prev->next = new_node; // 挂在冲突链条末尾
|
|
}
|
|
|
|
self->total_unique++;
|
|
return new_node;
|
|
}
|
|
|
|
static c_err_t c_NlpCharNode_AddRelation(c_NlpCharEdge_t** edge_buckets, c_size_t capacity, c_ucs4_t target_cp) {
|
|
c_size_t idx = c_NlpGraph_Hash(target_cp, capacity);
|
|
c_NlpCharEdge_t* curr = edge_buckets[idx];
|
|
|
|
while (curr != NULL) {
|
|
if (curr->target_cp == target_cp) {
|
|
curr->co_count++;
|
|
return C_ERR_OK;
|
|
}
|
|
curr = curr->next;
|
|
}
|
|
|
|
c_NlpCharEdge_t* new_edge = (c_NlpCharEdge_t*)C_ALLOC(sizeof(c_NlpCharEdge_t));
|
|
if (!new_edge) return C_ERR_NOMEM;
|
|
|
|
new_edge->target_cp = target_cp;
|
|
new_edge->co_count = 1;
|
|
new_edge->next = edge_buckets[idx];
|
|
edge_buckets[idx] = new_edge;
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------------------------------------ */
|
|
/* */
|
|
|
|
c_err_t c_NlpCharGraph_Init(c_NlpCharGraph_t* self, c_size_t bucket_count, c_size_t relation_default_capacity) {
|
|
if (!self || bucket_count == 0 || relation_default_capacity == 0) return C_ERR_PARAM;
|
|
|
|
self->bucket_count = bucket_count;
|
|
self->relation_default_capacity = relation_default_capacity;
|
|
self->total_unique = 0;
|
|
self->total_words = 0;
|
|
|
|
self->buckets = (c_NlpCharNode_t**)C_ALLOC(sizeof(c_NlpCharNode_t*) * bucket_count);
|
|
if (!self->buckets) return C_ERR_NOMEM;
|
|
|
|
for (c_size_t i = 0; i < bucket_count; i++) {
|
|
self->buckets[i] = NULL;
|
|
}
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
c_NlpCharNode_t* c_NlpCharGraph_GetNode(c_NlpCharGraph_t* self, c_ucs4_t cp) {
|
|
if (!self || !self->buckets) return NULL;
|
|
c_size_t idx = c_NlpGraph_Hash(cp, self->bucket_count);
|
|
c_NlpCharNode_t* curr = self->buckets[idx];
|
|
while (curr != NULL) {
|
|
if (curr->codepoint == cp) return curr;
|
|
curr = curr->next;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
c_err_t c_NlpCharGraph_Process(c_NlpCharGraph_t* self, const c_ucs4_t* unicode_array, c_size_t length) {
|
|
if (!self || !self->buckets || !unicode_array || length == 0) return C_ERR_PARAM;
|
|
|
|
self->total_words += length;
|
|
|
|
for (c_size_t i = 0; i < length; i++) {
|
|
c_ucs4_t curr_cp = unicode_array[i];
|
|
|
|
// 1. 通过拉链机制获取/创建当前节点
|
|
c_NlpCharNode_t* curr_node = c_NlpCharGraph_GetOrCreateNode(self, curr_cp);
|
|
if (!curr_node) return C_ERR_NOMEM;
|
|
|
|
curr_node->frequency++;
|
|
c_err_t err = c_NlpOffsetList_Add(&curr_node->offset_list, i);
|
|
if (err != C_ERR_OK) return err;
|
|
|
|
// 2. 建立前序边，同时保证前序字在拉链表中不丢失
|
|
if (i > 0) {
|
|
c_ucs4_t prev_cp = unicode_array[i - 1];
|
|
c_NlpCharNode_t* p_node = c_NlpCharGraph_GetOrCreateNode(self, prev_cp);
|
|
if (!p_node) return C_ERR_NOMEM;
|
|
|
|
err = c_NlpCharNode_AddRelation(curr_node->prev_chars, curr_node->relation_capacity, prev_cp);
|
|
if (err != C_ERR_OK) return err;
|
|
}
|
|
|
|
// 3. 建立后续边（如“大”关联“模”），同时通过拉链确保“模”字被注册激活
|
|
if (i < length - 1) {
|
|
c_ucs4_t next_cp = unicode_array[i + 1];
|
|
c_NlpCharNode_t* n_node = c_NlpCharGraph_GetOrCreateNode(self, next_cp);
|
|
if (!n_node) return C_ERR_NOMEM;
|
|
|
|
err = c_NlpCharNode_AddRelation(curr_node->next_chars, curr_node->relation_capacity, next_cp);
|
|
if (err != C_ERR_OK) return err;
|
|
}
|
|
}
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
|
|
static void c_NlpCharEdge_FreeBuckets(c_NlpCharEdge_t** edge_buckets, c_size_t capacity) {
|
|
if (!edge_buckets) return;
|
|
for (c_size_t i = 0; i < capacity; i++) {
|
|
c_NlpCharEdge_t* curr = edge_buckets[i];
|
|
while (curr != NULL) {
|
|
c_NlpCharEdge_t* temp = curr->next;
|
|
C_FREE(curr);
|
|
curr = temp;
|
|
}
|
|
}
|
|
}
|
|
|
|
void c_NlpCharGraph_Destroy(c_NlpCharGraph_t* self) {
|
|
if (!self || !self->buckets || self->bucket_count==0) return;
|
|
|
|
for (c_size_t i = 0; i < self->bucket_count; i++) {
|
|
c_NlpCharNode_t* node = self->buckets[i];
|
|
|
|
// 沿拉链释放所有冲突节点的深层堆空间
|
|
while (node != NULL) {
|
|
c_NlpCharNode_t* next_node = node->next; // 暂存下一个指针
|
|
|
|
C_FREE(node->offset_list.offsets);
|
|
c_NlpCharEdge_FreeBuckets(node->next_chars, node->relation_capacity);
|
|
c_NlpCharEdge_FreeBuckets(node->prev_chars, node->relation_capacity);
|
|
|
|
C_FREE(node->next_chars);
|
|
C_FREE(node->prev_chars);
|
|
C_FREE(node);
|
|
|
|
node = next_node;
|
|
}
|
|
self->buckets[i] = NULL;
|
|
}
|
|
C_FREE(self->buckets);
|
|
self->bucket_count = 0;
|
|
self->total_unique = 0;
|
|
self->total_words = 0;
|
|
}
|
|
|
|
|
|
/* ------------------------------------------------------------------------------------------------------------------ */
|
|
/* */
|
|
|
|
|
|
c_err_t c_NlpCharGraph_AnalyzeDistanceWindow(c_NlpCharGraph_t* self, c_ucs4_t cp1, c_ucs4_t cp2, c_size_t window_size,
|
|
c_size_t* out_min_dist, double* out_avg_dist, c_size_t* out_co_occurrences,
|
|
c_size_t* out_cp1_precedes_count, c_size_t* out_cp2_precedes_count) {
|
|
// 1. Defend against parameter null configurations or invalid window constraints
|
|
if (!self || !out_min_dist || !out_avg_dist || !out_co_occurrences ||
|
|
!out_cp1_precedes_count || !out_cp2_precedes_count || window_size == 0) {
|
|
return C_ERR_PARAM;
|
|
}
|
|
|
|
// Initialize all metrics to zero out out-parameters safely
|
|
*out_min_dist = (c_size_t)-1;
|
|
*out_avg_dist = 0.0;
|
|
*out_co_occurrences = 0;
|
|
*out_cp1_precedes_count = 0;
|
|
*out_cp2_precedes_count = 0;
|
|
|
|
// 2. Fetch the structural node contexts from the master hash bucket allocations
|
|
c_NlpCharNode_t* n1 = c_NlpCharGraph_GetNode(self, cp1);
|
|
c_NlpCharNode_t* n2 = c_NlpCharGraph_GetNode(self, cp2);
|
|
if (!n1 || !n2 || n1->offset_list.count == 0 || n2->offset_list.count == 0) {
|
|
return C_ERR_OK; // Return cleanly with zeroed structures if any token is absent
|
|
}
|
|
|
|
c_size_t* o1 = n1->offset_list.offsets;
|
|
c_size_t* o2 = n2->offset_list.offsets;
|
|
c_size_t len1 = n1->offset_list.count;
|
|
c_size_t len2 = n2->offset_list.count;
|
|
|
|
c_size_t min_dist = (c_size_t)-1;
|
|
double total_dist_sum = 0.0;
|
|
c_size_t window_crossings = 0;
|
|
c_size_t cp1_precedes = 0;
|
|
c_size_t cp2_precedes = 0;
|
|
|
|
// 3. Deploy the Bounded Look-Ahead Sliding Scanner
|
|
// Loop through every single occurrence of cp1
|
|
for (c_size_t i = 0; i < len1; i++) {
|
|
c_size_t pos1 = o1[i];
|
|
|
|
// Find the first index in o2 where the element might be in range: pos2 >= pos1 - window_size
|
|
// We initialize look_idx using a basic optimization strategy, or start from 0
|
|
c_size_t look_idx = 0;
|
|
|
|
while (look_idx < len2) {
|
|
c_size_t pos2 = o2[look_idx];
|
|
|
|
// Condition A: pos2 is too far behind pos1, keep advancing the scan
|
|
if (pos2 < pos1 && (pos1 - pos2) > window_size) {
|
|
look_idx++;
|
|
continue;
|
|
}
|
|
|
|
// Condition B: pos2 is too far ahead of pos1, we can stop evaluating for this pos1
|
|
if (pos2 > pos1 && (pos2 - pos1) > window_size) {
|
|
break;
|
|
}
|
|
|
|
// Condition C: pos2 falls within the safe sliding context window boundary limit!
|
|
c_size_t dist = (pos1 > pos2) ? (pos1 - pos2) : (pos2 - pos1);
|
|
|
|
if (dist < min_dist) {
|
|
min_dist = dist;
|
|
}
|
|
|
|
total_dist_sum += (double)dist;
|
|
window_crossings++;
|
|
|
|
if (pos1 < pos2) {
|
|
cp1_precedes++; // cp1 appears BEFORE cp2
|
|
} else if (pos1 > pos2) {
|
|
cp2_precedes++; // cp2 appears BEFORE cp1
|
|
} else {
|
|
// Identity overlaps count as neutral crossings (rare for unique terms)
|
|
}
|
|
|
|
look_idx++;
|
|
}
|
|
}
|
|
|
|
// 4. Populate analytics result payloads accurately
|
|
if (window_crossings > 0) {
|
|
*out_min_dist = min_dist;
|
|
*out_avg_dist = total_dist_sum / (double)window_crossings;
|
|
*out_co_occurrences = window_crossings;
|
|
*out_cp1_precedes_count = cp1_precedes;
|
|
*out_cp2_precedes_count = cp2_precedes;
|
|
}
|
|
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------------------------------------ */
|
|
/* */
|
|
|
|
c_err_t c_NlpCharGraph_ExportJSON(c_NlpCharGraph_t* self, const char* filepath) {
|
|
if (!self || !self->buckets || !filepath) return C_ERR_PARAM;
|
|
|
|
c_StringBuffer_t sb;
|
|
c_err_t err = c_StringBuffer_Init(&sb, 4096);
|
|
if (err != C_ERR_OK) return err;
|
|
|
|
err = c_StringBuffer_AppendStr(&sb, "{\n");
|
|
err |= c_StringBuffer_AppendStr(&sb, " \"meta\": {\n");
|
|
|
|
char meta_buf[128];
|
|
snprintf(meta_buf, sizeof(meta_buf), " \"total_words\": %lu,\n \"total_unique\": %lu\n",
|
|
(unsigned long)self->total_words, (unsigned long)self->total_unique);
|
|
err |= c_StringBuffer_AppendStr(&sb, meta_buf);
|
|
err |= c_StringBuffer_AppendStr(&sb, " },\n");
|
|
err |= c_StringBuffer_AppendStr(&sb, " \"nodes\": [\n");
|
|
if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
|
|
|
|
c_size_t nodes_serialized = 0;
|
|
|
|
// 外层遍历所有桶
|
|
for (c_size_t i = 0; i < self->bucket_count; i++) {
|
|
c_NlpCharNode_t* node = self->buckets[i];
|
|
|
|
// 核心修正：内层沿着冲突拉链链表深度追踪，把每一个被掩埋的字（如“模”）都全部挖出来导出
|
|
while (node != NULL) {
|
|
char utf8_word[8];
|
|
c_size_t written_len = 0;
|
|
if (c_utf8_from_unicode(node->codepoint, utf8_word, &written_len) != C_ERR_OK) {
|
|
node = node->next;
|
|
continue;
|
|
}
|
|
|
|
if (nodes_serialized > 0) {
|
|
err |= c_StringBuffer_AppendStr(&sb, ",\n");
|
|
}
|
|
|
|
err |= c_StringBuffer_AppendStr(&sb, " {\n");
|
|
|
|
char node_info[256];
|
|
if (utf8_word[0] == '"' || utf8_word[0] == '\\') {
|
|
snprintf(node_info, sizeof(node_info),
|
|
" \"char\": \"\\%s\",\n \"codepoint\": %u,\n \"frequency\": %lu,\n",
|
|
utf8_word, node->codepoint, (unsigned long)node->frequency);
|
|
} else {
|
|
snprintf(node_info, sizeof(node_info),
|
|
" \"char\": \"%s\",\n \"codepoint\": %u,\n \"frequency\": %lu,\n",
|
|
utf8_word, node->codepoint, (unsigned long)node->frequency);
|
|
}
|
|
err |= c_StringBuffer_AppendStr(&sb, node_info);
|
|
|
|
// 序列化 offsets
|
|
err |= c_StringBuffer_AppendStr(&sb, " \"offsets\": [");
|
|
for (c_size_t k = 0; k < node->offset_list.count; k++) {
|
|
char val_buf[32];
|
|
snprintf(val_buf, sizeof(val_buf), "%lu%s",
|
|
(unsigned long)node->offset_list.offsets[k],
|
|
(k + 1 < node->offset_list.count) ? ", " : "");
|
|
err |= c_StringBuffer_AppendStr(&sb, val_buf);
|
|
}
|
|
err |= c_StringBuffer_AppendStr(&sb, "],\n");
|
|
|
|
// 序列化 next_chars
|
|
err |= c_StringBuffer_AppendStr(&sb, " \"next_chars\": {");
|
|
c_bool_t edge_written = C_FALSE;
|
|
|
|
for (c_size_t bucket_idx = 0; bucket_idx < node->relation_capacity; bucket_idx++) {
|
|
c_NlpCharEdge_t* edge = node->next_chars[bucket_idx];
|
|
while (edge != NULL) {
|
|
char edge_utf8[8];
|
|
c_size_t e_len = 0;
|
|
if (c_utf8_from_unicode(edge->target_cp, edge_utf8, &e_len) == C_ERR_OK) {
|
|
char edge_buf[128];
|
|
const char* comma_prefix = edge_written ? ", " : "";
|
|
|
|
if (edge_utf8[0] == '"' || edge_utf8[0] == '\\') {
|
|
snprintf(edge_buf, sizeof(edge_buf), "%s\"\\%s\": %lu",
|
|
comma_prefix, edge_utf8, (unsigned long)edge->co_count);
|
|
} else {
|
|
snprintf(edge_buf, sizeof(edge_buf), "%s\"%s\": %lu",
|
|
comma_prefix, edge_utf8, (unsigned long)edge->co_count);
|
|
}
|
|
err |= c_StringBuffer_AppendStr(&sb, edge_buf);
|
|
edge_written = C_TRUE;
|
|
}
|
|
edge = edge->next;
|
|
}
|
|
}
|
|
err |= c_StringBuffer_AppendStr(&sb, "}\n");
|
|
err |= c_StringBuffer_AppendStr(&sb, " }");
|
|
|
|
nodes_serialized++;
|
|
|
|
// 顺着冲突拉链移动到下一个发生哈希冲突的字节点
|
|
node = node->next;
|
|
}
|
|
if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
|
|
}
|
|
|
|
err |= c_StringBuffer_AppendStr(&sb, "\n ]\n}\n");
|
|
if (err != C_ERR_OK) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
|
|
|
|
FILE* file = fopen(filepath, "wb");
|
|
if (!file) { c_StringBuffer_Destroy(&sb); return C_ERR_FAIL; }
|
|
|
|
c_size_t expected_size = sb.size;
|
|
size_t written = fwrite(sb.buffer, 1, expected_size, file);
|
|
fclose(file);
|
|
c_StringBuffer_Destroy(&sb);
|
|
|
|
if (written != expected_size) return C_ERR_FAIL;
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
|
|
/* ------------------------------------------------------------------------------------------------------------------ */
|
|
/* */
|
|
|
|
/**
|
|
* @brief Helper utility to safely extract value payloads trapped between specific JSON keys and brackets.
|
|
* Example: transforms ' "char": "大", ' into '大'
|
|
*/
|
|
static void c_Nlp_ExtractJsonValue(const char* line, const char* key, char* dest_val, size_t dest_max) {
|
|
dest_val[0] = '\0';
|
|
char* key_pos = strstr(line, key);
|
|
if (!key_pos) return;
|
|
|
|
char* start = strchr(key_pos + strlen(key), ':');
|
|
if (!start) return;
|
|
start++; // Step past ':'
|
|
|
|
// Skip white spaces and opening quotes
|
|
while (*start == ' ' || *start == '\t' || *start == '"') {
|
|
start++;
|
|
}
|
|
|
|
size_t idx = 0;
|
|
while (*start != '\0' && *start != '"' && *start != ',' && *start != '\n' && *start != '\r' && *start != ']' && idx < dest_max - 1) {
|
|
// Handle escaped JSON string properties
|
|
if (*start == '\\' && *(start + 1) != '\0') {
|
|
start++;
|
|
}
|
|
dest_val[idx++] = *start++;
|
|
}
|
|
dest_val[idx] = '\0';
|
|
|
|
// Trim trailing tracking spaces if any present
|
|
while (idx > 0 && (dest_val[idx - 1] == ' ' || dest_val[idx - 1] == '\t' || dest_val[idx - 1] == '}')) {
|
|
dest_val[--idx] = '\0';
|
|
}
|
|
}
|
|
|
|
|
|
|
|
c_err_t c_NlpCharGraph_ImportJSON(c_NlpCharGraph_t* self, const char* filepath, c_size_t bucket_count) {
|
|
if (!self || !filepath || bucket_count == 0) return C_ERR_PARAM;
|
|
|
|
FILE* file = fopen(filepath, "rb");
|
|
if (!file) return C_ERR_FAIL;
|
|
|
|
// Detect and safely strip standard UTF-8 BOM if present
|
|
unsigned char check_bom[3];
|
|
if (fread(check_bom, 1, 3, file) == 3 && check_bom[0] == 0xEF && check_bom[1] == 0xBB && check_bom[2] == 0xBF) {
|
|
// BOM skipped successfully
|
|
} else {
|
|
fseek(file, 0, SEEK_SET);
|
|
}
|
|
|
|
c_StringBuffer_t line_sb;
|
|
c_err_t err = c_StringBuffer_Init(&line_sb, 512);
|
|
if (err != C_ERR_OK) {
|
|
fclose(file);
|
|
return err;
|
|
}
|
|
|
|
c_size_t total_words = 0;
|
|
c_size_t relation_cap = 16; // Default initialization relation table width
|
|
|
|
// Fixed Buffer Upgrade: Secure 4096-byte stack arrays to completely negate truncation risks
|
|
char val_buf[4096];
|
|
|
|
// Phase 1: Parse Global Metadata Attributes
|
|
while (c_utf8_file_readline(file, &line_sb) == C_ERR_OK) {
|
|
if (strstr(line_sb.buffer, "\"total_words\"") != NULL) {
|
|
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"total_words\"", val_buf, sizeof(val_buf));
|
|
total_words = (c_size_t)strtoul(val_buf, NULL, 10);
|
|
}
|
|
if (strstr(line_sb.buffer, "\"nodes\"") != NULL) {
|
|
break; // Metadata block parsed, moving onto array entries
|
|
}
|
|
}
|
|
|
|
// Phase 2: Structural Graph Memory Instantiation
|
|
err = c_NlpCharGraph_Init(self, bucket_count, relation_cap);
|
|
if (err != C_ERR_OK) {
|
|
c_StringBuffer_Destroy(&line_sb);
|
|
fclose(file);
|
|
return err;
|
|
}
|
|
self->total_words = total_words;
|
|
|
|
c_NlpCharNode_t* active_node = NULL;
|
|
|
|
// Phase 3: Sequential Structural Node Deserialization
|
|
while (c_utf8_file_readline(file, &line_sb) == C_ERR_OK) {
|
|
// Ignore standalone array boundaries markers
|
|
if (strstr(line_sb.buffer, "{") != NULL && strstr(line_sb.buffer, "\"char\"") == NULL) {
|
|
continue;
|
|
}
|
|
|
|
// Parse individual token parameters
|
|
if (strstr(line_sb.buffer, "\"codepoint\"") != NULL) {
|
|
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"codepoint\"", val_buf, sizeof(val_buf));
|
|
c_ucs4_t cp = (c_ucs4_t)strtoul(val_buf, NULL, 10);
|
|
|
|
// Reconstruct utilizing standard chaining allocator to secure the linked pointers list
|
|
active_node = c_NlpCharGraph_GetOrCreateNode(self, cp);
|
|
if (!active_node) {
|
|
c_NlpCharGraph_Destroy(self);
|
|
c_StringBuffer_Destroy(&line_sb);
|
|
fclose(file);
|
|
return C_ERR_NOMEM;
|
|
}
|
|
}
|
|
else if (strstr(line_sb.buffer, "\"frequency\"") != NULL && active_node) {
|
|
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"frequency\"", val_buf, sizeof(val_buf));
|
|
active_node->frequency = (c_size_t)strtoul(val_buf, NULL, 10);
|
|
}
|
|
else if (strstr(line_sb.buffer, "\"offsets\"") != NULL && active_node) {
|
|
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"offsets\"", val_buf, sizeof(val_buf));
|
|
|
|
// Safe continuous array chunk tokenization using reentrant strtok mechanics
|
|
char* save_ptr = NULL;
|
|
char* token = c_utf8_strtok(val_buf, ", ", &save_ptr);
|
|
while (token != NULL) {
|
|
if (*token != '\0') {
|
|
c_size_t offset_val = (c_size_t)strtoul(token, NULL, 10);
|
|
c_NlpOffsetList_Add(&active_node->offset_list, offset_val);
|
|
}
|
|
token = c_utf8_strtok(NULL, ", ", &save_ptr);
|
|
}
|
|
}
|
|
else if (strstr(line_sb.buffer, "\"next_chars\"") != NULL && active_node) {
|
|
c_Nlp_ExtractJsonValue(line_sb.buffer, "\"next_chars\"", val_buf, sizeof(val_buf));
|
|
|
|
// Process directional map entries: e.g., '"I": 1, "型": 1'
|
|
char* save_ptr = NULL;
|
|
char* pair_token = c_utf8_strtok(val_buf, ", ", &save_ptr);
|
|
while (pair_token != NULL) {
|
|
char* colon = strchr(pair_token, ':');
|
|
if (colon) {
|
|
*colon = '\0';
|
|
char edge_char[4096]; // Secure localized tracking array width
|
|
strncpy(edge_char, pair_token, sizeof(edge_char) - 1);
|
|
edge_char[sizeof(edge_char) - 1] = '\0';
|
|
|
|
// Strip enclosing quote strings out of target boundaries
|
|
char* edge_clean = edge_char;
|
|
while (*edge_clean == ' ' || *edge_clean == '\t' || *edge_clean == '"') {
|
|
edge_clean++;
|
|
}
|
|
size_t elen = strlen(edge_clean);
|
|
while (elen > 0 && (edge_clean[elen - 1] == '"' || edge_clean[elen - 1] == ' ' || edge_clean[elen - 1] == '\t')) {
|
|
edge_clean[--elen] = '\0';
|
|
}
|
|
|
|
if (elen > 0) {
|
|
c_ucs4_t target_cp = 0;
|
|
c_size_t bytes_step = 0;
|
|
// Transform character text string representation straight back into standard 32-bit integer points
|
|
if (c_utf8_to_unicode(edge_clean, &target_cp, &bytes_step) == C_ERR_OK) {
|
|
c_size_t edge_freq = (c_size_t)strtoul(colon + 1, NULL, 10);
|
|
|
|
// Rehydrate co-occurrence link matrices
|
|
for (c_size_t f = 0; f < edge_freq; f++) {
|
|
c_NlpCharNode_AddRelation(active_node->next_chars, active_node->relation_capacity, target_cp);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
pair_token = c_utf8_strtok(NULL, ", ", &save_ptr);
|
|
}
|
|
}
|
|
}
|
|
|
|
c_StringBuffer_Destroy(&line_sb);
|
|
fclose(file);
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
/* ------------------------------------------------------------------------------------------------------------------ */
|
|
/* */
|
|
|
|
c_err_t c_NlpCharGraph_PredictNext(c_NlpCharGraph_t* self, c_ucs4_t current_char, c_NlpPredictMode_t mode,
|
|
c_ucs4_t* out_next_char, double* out_confidence) {
|
|
if (!self || !out_next_char || !out_confidence) {
|
|
return C_ERR_PARAM;
|
|
}
|
|
|
|
// 1. 在主哈希拉链表中定位当前字的节点
|
|
c_NlpCharNode_t* node = c_NlpCharGraph_GetNode(self, current_char);
|
|
if (!node || node->frequency == 0) {
|
|
return C_ERR_FAIL; // 图中不存在这个字，或者该字从未作为主词出现过
|
|
}
|
|
|
|
// 2. 遍历该节点的所有 next_chars 边，统计后续字的总权重和最大频次
|
|
c_size_t total_co_count = 0;
|
|
c_size_t max_co_count = 0;
|
|
c_ucs4_t best_target_cp = 0;
|
|
|
|
for (c_size_t i = 0; i < node->relation_capacity; i++) {
|
|
c_NlpCharEdge_t* edge = node->next_chars[i];
|
|
while (edge != NULL) {
|
|
total_co_count += edge->co_count;
|
|
|
|
// 贪婪模式所需的条件：记录最大频次
|
|
if (edge->co_count > max_co_count) {
|
|
max_co_count = edge->co_count;
|
|
best_target_cp = edge->target_cp;
|
|
}
|
|
edge = edge->next;
|
|
}
|
|
}
|
|
|
|
// 如果该字没有任何后续关联（比如它是文章的最后一个字且仅出现了一次）
|
|
if (total_co_count == 0) {
|
|
return C_ERR_FAIL;
|
|
}
|
|
|
|
// 3. 根据预测模式确定输出
|
|
if (mode == C_PREDICT_MODE_GREEDY) {
|
|
// 【贪婪模式】：直接选择后序关联最强、出现次数最多的字
|
|
*out_next_char = best_target_cp;
|
|
*out_confidence = (double)max_co_count / (double)total_co_count;
|
|
}
|
|
else if (mode == C_PREDICT_MODE_SAMPLING) {
|
|
// 【随机抽样模式】：基于一阶马尔可夫链的条件概率分布进行轮盘赌抽样
|
|
// 生成一个 0.0 到 1.0 之间的随机数
|
|
double random_pivot = (double)rand() / (double)RAND_MAX;
|
|
double current_accumulator = 0.0;
|
|
c_bool_t sampled = C_FALSE;
|
|
|
|
for (c_size_t i = 0; i < node->relation_capacity; i++) {
|
|
c_NlpCharEdge_t* edge = node->next_chars[i];
|
|
while (edge != NULL) {
|
|
double probability = (double)edge->co_count / (double)total_co_count;
|
|
current_accumulator += probability;
|
|
|
|
// 落在当前随机区间内，命中该字符
|
|
if (random_pivot <= current_accumulator) {
|
|
*out_next_char = edge->target_cp;
|
|
*out_confidence = probability;
|
|
sampled = C_TRUE;
|
|
break;
|
|
}
|
|
edge = edge->next;
|
|
}
|
|
if (sampled) break;
|
|
}
|
|
|
|
// 兜底保护，防止浮点数精度截断导致未命中
|
|
if (!sampled) {
|
|
*out_next_char = best_target_cp;
|
|
*out_confidence = (double)max_co_count / (double)total_co_count;
|
|
}
|
|
} else {
|
|
return C_ERR_PARAM;
|
|
}
|
|
|
|
return C_ERR_OK;
|
|
}
|
|
|