pchen
e6a29bcdfd
c_NlpNgramGraph: 基于 n 个已经出现的词，预估后续可能出现的词 c_NlpLanguageModel: 如果有的词组合形式没有出现过，找到之前学习过最长类似的词句进行预测
190 lines
9.3 KiB
C
190 lines
9.3 KiB
C
#include "c_NlpCharGraph.h"
|
||
#include <stdlib.h>
|
||
#include <stdio.h>
|
||
|
||
#include "c_StringBuffer.h"
|
||
#include "c_utf8_file.h"
|
||
|
||
#define RUN_TEST(test_case, name) \
|
||
do { \
|
||
printf("[RUN] %s... ", name); \
|
||
if (test_case) { \
|
||
printf("\033[32mPASSED\033[0m\n"); \
|
||
} else { \
|
||
printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \
|
||
return C_ERR_FAIL; \
|
||
} \
|
||
} while(0)
|
||
|
||
static c_err_t test(void) {
|
||
/* 29. Arbitrary-Width Character Relation Graph Testing */
|
||
c_NlpCharGraph_t var_graph;
|
||
|
||
// Initialize graph with 512 main buckets and an arbitrary custom relation layout size of 128 channels
|
||
c_err_t err = c_NlpCharGraph_Init(&var_graph, 512, 128);
|
||
RUN_TEST(err == C_ERR_OK, "Variable-width graph initialization returns C_ERR_OK");
|
||
|
||
c_ucs4_t sample_text[] = { 0x540C, 0x6B65, 0x540C }; // '同', '步', '同'
|
||
err = c_NlpCharGraph_Process(&var_graph, sample_text, 3);
|
||
RUN_TEST(err == C_ERR_OK && var_graph.total_unique == 2, "Processed text array inside flexible capacity container");
|
||
|
||
c_NlpCharNode_t* node_tong = c_NlpCharGraph_GetNode(&var_graph, 0x540C); // '同'
|
||
RUN_TEST(node_tong != NULL && node_tong->relation_capacity == 128, "Verified node dynamically inherited the arbitrary 128-bucket relationship size setting");
|
||
RUN_TEST(node_tong->next_chars != NULL && node_tong->prev_chars != NULL, "relational heap allocation arrays successfully bound");
|
||
c_NlpCharGraph_Destroy(&var_graph);
|
||
|
||
/* ------------------------------------------------------------------------------------------------------------------ */
|
||
/* */
|
||
|
||
/* 30. Corrected Context Window Proximity Analytical Verification */
|
||
c_NlpCharGraph_t win_graph;
|
||
c_NlpCharGraph_Init(&win_graph, 1024, C_UNICODE_MAX);
|
||
|
||
// Corpus structural token layout: "A B B C A" -> Test multi-point matching
|
||
// Let 0x0041 = 'A', 0x0042 = 'B'
|
||
c_ucs4_t cluster_corpus[] = { 0x0041, 0x0042, 0x0042, 0x0043, 0x0041 };
|
||
err = c_NlpCharGraph_Process(&win_graph, cluster_corpus, 5);
|
||
RUN_TEST(err == C_ERR_OK, "Cluster corpus parsed inside window context analyzer container");
|
||
|
||
c_size_t min_d = 0;
|
||
double avg_d = 0.0;
|
||
c_size_t cross_count = 0;
|
||
c_size_t count_precedes_1 = 0;
|
||
c_size_t count_precedes_2 = 0;
|
||
|
||
// Analyze token proximity between 'A' (Index 0, 4) and 'B' (Index 1, 2) with a window threshold size of 3
|
||
// Target matches:
|
||
// For A at 0: B at 1 (dist 1, A precedes), B at 2 (dist 2, A precedes) -> 2 pairs
|
||
// For A at 4: B at 1 (dist 3, B precedes), B at 2 (dist 2, B precedes) -> 2 pairs
|
||
// Total co-occurrences expected = 4 crossings. Min distance = 1.
|
||
// Total distance sum = 1 + 2 + 3 + 2 = 8. Avg distance = 8 / 4 = 2.0.
|
||
err = c_NlpCharGraph_AnalyzeDistanceWindow(&win_graph, 0x0041, 0x0042, 3,
|
||
&min_d, &avg_d, &cross_count,
|
||
&count_precedes_1, &count_precedes_2);
|
||
|
||
RUN_TEST(err == C_ERR_OK, "Corrected window distance execution completed successfully");
|
||
RUN_TEST(cross_count == 4, "Accurately extracted exactly 4 window crossing combinations across the cluster");
|
||
RUN_TEST(min_d == 1, "Accurately isolated absolute minimum distance down to 1 token ('A' at 0 vs 'B' at 1)");
|
||
RUN_TEST(avg_d == 2.0, "Accurately evaluated average mathematical mean distance at 2.0 tokens precisely");
|
||
RUN_TEST(count_precedes_1 == 2, "Verified 'A' preceded 'B' exactly 2 times across window evaluation frames");
|
||
RUN_TEST(count_precedes_2 == 2, "Verified 'B' preceded 'A' exactly 2 times across window evaluation frames");
|
||
|
||
c_NlpCharGraph_Destroy(&win_graph);
|
||
|
||
/* ------------------------------------------------------------------------------------------------------------------ */
|
||
/* */
|
||
|
||
/* 31. Unicode Character Graph JSON Export Verification */
|
||
c_NlpCharGraph_t json_graph;
|
||
const char* output_json_path = "nlp_graph_export.json";
|
||
|
||
// Initialize graph context
|
||
err = c_NlpCharGraph_Init(&json_graph, 16, 4);
|
||
RUN_TEST(err == C_ERR_OK, "JSON Graph context initialized successfully");
|
||
|
||
// Input string: "AI大模型" -> Translated into explicit unicode codepoint stream arrays
|
||
c_ucs4_t export_corpus[] = { 0x0041, 0x0049, 0x5927, 0x6A21, 0x578B };
|
||
err = c_NlpCharGraph_Process(&json_graph, export_corpus, 5);
|
||
RUN_TEST(err == C_ERR_OK, "Export corpus strings parsed smoothly inside graph matrices");
|
||
|
||
// Execute JSON Export API pipeline
|
||
err = c_NlpCharGraph_ExportJSON(&json_graph, output_json_path);
|
||
RUN_TEST(err == C_ERR_OK, "c_NlpCharGraph_ExportJSON successfully generated JSON file on disk");
|
||
|
||
// Read file back to verify it contains valid structural characters
|
||
c_StringBuffer_t read_verify_sb;
|
||
c_StringBuffer_Init(&read_verify_sb, 1024);
|
||
err = c_utf8_file_read(output_json_path, &read_verify_sb);
|
||
|
||
RUN_TEST(err == C_ERR_OK, "Successfully read generated JSON output file back into workspace");
|
||
RUN_TEST(strstr(read_verify_sb.buffer, "\"total_words\": 5") != NULL, "JSON metadata section successfully parsed");
|
||
RUN_TEST(strstr(read_verify_sb.buffer, "\"char\": \"大\"") != NULL, "Unicode codepoint 0x5927 successfully serialized to UTF-8 character string '大'");
|
||
RUN_TEST(strstr(read_verify_sb.buffer, "\"next_chars\":") != NULL, "Directional adjacency edge transition attributes successfully embedded");
|
||
|
||
// Clean up temporary files and memory structures
|
||
c_StringBuffer_Destroy(&read_verify_sb);
|
||
c_NlpCharGraph_Destroy(&json_graph);
|
||
|
||
/* ------------------------------------------------------------------------------------------------------------------ */
|
||
/* */
|
||
|
||
/* 32. Unicode Character Graph JSON Import Validation */
|
||
c_NlpCharGraph_t imported_graph;
|
||
const char* target_json_src = "nlp_graph_export.json";
|
||
|
||
// Dependencies checkpoint: Ensure file exists before running verification passes
|
||
// (Generated by calling c_NlpCharGraph_ExportJSON in the previous step)
|
||
err = c_NlpCharGraph_ImportJSON(&imported_graph, target_json_src, 1024);
|
||
RUN_TEST(err == C_ERR_OK, "c_NlpCharGraph_ImportJSON parsed JSON text stream and built the graph successfully");
|
||
|
||
// Validate Structural Reconstitution Fidelity Metrics
|
||
RUN_TEST(imported_graph.total_words == 5, "Reconstructed metadata 'total_words' matches value: 5");
|
||
|
||
// Fetch and check nodes internally
|
||
c_NlpCharNode_t* check_da = c_NlpCharGraph_GetNode(&imported_graph, 0x5927); // '大'
|
||
RUN_TEST(check_da != NULL, "Unicode Node 0x5927 ('大') successfully verified in bucket storage");
|
||
if (check_da) {
|
||
RUN_TEST(check_da->frequency == 1, "Frequency counter holds correct historical state value: 1");
|
||
RUN_TEST(check_da->offset_list.count == 1, "Offsets dynamic list successfully loaded elements");
|
||
|
||
// Trace and check directional bigram linkages
|
||
c_size_t min_d = 0;
|
||
double avg_d = 0.0;
|
||
c_size_t crossings = 0;
|
||
c_size_t p1 = 0, p2 = 0;
|
||
err = c_NlpCharGraph_AnalyzeDistanceWindow(&imported_graph, 0x5927, 0x6A21, 2, &min_d, &avg_d, &crossings, &p1, &p2); // '大' vs '模'
|
||
RUN_TEST(err == C_ERR_OK && crossings == 1, "Relational bigram matching layers successfully operational post-import");
|
||
}
|
||
|
||
c_NlpCharGraph_Destroy(&imported_graph);
|
||
|
||
// remove(output_json_path); // Remove transient file from disk space
|
||
|
||
/* ------------------------------------------------------------------------------------------------------------------ */
|
||
/* */
|
||
|
||
/* 31. 语言模型后续字预测验证 (c_NlpCharGraph_PredictNext) */
|
||
c_NlpCharGraph_t pred_graph;
|
||
c_NlpCharGraph_Init(&pred_graph, 1024, 16);
|
||
|
||
// 载入高重复、高确定性的测试语料："大模型大模型大项目"
|
||
// '大' 后面出现了 2 次 '模'，1 次 '项'。总后续数 = 3
|
||
c_ucs4_t train_data[] = {
|
||
0x5927, 0x6A21, 0x578B, // 大模型
|
||
0x5927, 0x6A21, 0x578B, // 大模型
|
||
0x5927, 0x9879, 0x76EE // 大项目
|
||
};
|
||
err = c_NlpCharGraph_Process(&pred_graph, train_data, 9);
|
||
RUN_TEST(err == C_ERR_OK, "Prediction training data parsed smoothly");
|
||
|
||
c_ucs4_t predicted_char = 0;
|
||
double confidence = 0.0;
|
||
|
||
// 进行确定性的贪婪模式预测：输入 '大' (0x5927)
|
||
// 预期：'模' (0x6A21) 出现的概率是 2/3 = 66.67%，远高于 '项' 的 1/3
|
||
err = c_NlpCharGraph_PredictNext(&pred_graph, 0x5927, C_PREDICT_MODE_GREEDY, &predicted_char, &confidence);
|
||
|
||
RUN_TEST(err == C_ERR_OK, "PredictNext completed execution path successfully");
|
||
RUN_TEST(predicted_char == 0x6A21, "Greedy mode correctly deterministically chooses '模' following '大'");
|
||
// 校验置信度 2.0 / 3.0 近似等于 0.666667
|
||
RUN_TEST(confidence > 0.66 && confidence < 0.67, "Confidence value evaluated exactly at 66.67%%");
|
||
|
||
// 测试生僻词/无后续关联的边界安全
|
||
err = c_NlpCharGraph_PredictNext(&pred_graph, 0x76EE, C_PREDICT_MODE_GREEDY, &predicted_char, &confidence); // '目' 是文章末尾
|
||
RUN_TEST(err == C_ERR_FAIL, "PredictNext safely rejects character with zero trailing context links");
|
||
|
||
c_NlpCharGraph_Destroy(&pred_graph);
|
||
|
||
|
||
/* ------------------------------------------------------------------------------------------------------------------ */
|
||
/* */
|
||
|
||
return C_ERR_OK;
|
||
}
|
||
|
||
int main(int argc, char** argv){
|
||
return test();
|
||
|
||
return 0;
|
||
}
|