pchen
e6a29bcdfd
c_NlpNgramGraph: 基于 n 个已经出现的词，预估后续可能出现的词 c_NlpLanguageModel: 如果有的词组合形式没有出现过，找到之前学习过最长类似的词句进行预测
59 lines
2.7 KiB
C
59 lines
2.7 KiB
C
#include "c_NlpLanguageModel.h"
|
|
#include <stdlib.h>
|
|
#include <stdio.h>
|
|
|
|
#define RUN_TEST(test_case, name) \
|
|
do { \
|
|
printf("[RUN] %s... ", name); \
|
|
if (test_case) { \
|
|
printf("\033[32mPASSED\033[0m\n"); \
|
|
} else { \
|
|
printf("\033[31mFAILED\033[0m (%s:%d)\n", __FILE__, __LINE__); \
|
|
return C_ERR_FAIL; \
|
|
} \
|
|
} while(0)
|
|
|
|
static c_err_t test(void) {
|
|
/* 34. Katz Back-off Smoothing Cascade Verification */
|
|
c_NlpLanguageModel_t lm;
|
|
// Instantiate a 3-tier system (Trigram + Bigram cascaded backend framework mesh)
|
|
c_err_t err = c_NlpLanguageModel_Init(&lm, 3, 512);
|
|
RUN_TEST(err == C_ERR_OK, "Cascaded Language Model successfully initialized");
|
|
|
|
// Training set containing: "大模型" and "微模型"
|
|
c_ucs4_t text_stream[] = {
|
|
0x5927, 0x6A21, 0x578B, /* 大模型 */
|
|
0x5FAE, 0x6A21, 0x578B /* 微模型 */
|
|
};
|
|
err = c_NlpLanguageModel_Train(&lm, text_stream, 6);
|
|
RUN_TEST(err == C_ERR_OK, "Cascaded models successfully ingested corpus dataset streams");
|
|
|
|
c_ucs4_t next_prediction = 0;
|
|
double confidence_score = 0.0;
|
|
|
|
// --- Scenario A: Perfect high-order match ---
|
|
// Query context: "大模" -> Expected result: "型" (100% confidence from Trigram tier)
|
|
c_ucs4_t perfect_history[] = { 0x5927, 0x6A21 };
|
|
err = c_NlpLanguageModel_PredictSmooth(&lm, perfect_history, 2, &next_prediction, &confidence_score);
|
|
RUN_TEST(err == C_ERR_OK && next_prediction == 0x578B, "Scenario A: High-order match correctly resolves to '型'");
|
|
RUN_TEST(confidence_score == 1.0, "Scenario A: Direct hit returns full unpenalized confidence metrics");
|
|
|
|
// --- Scenario B: Unseen Trigram history forces a Back-off step ---
|
|
// Query context: "新模" -> The 3-gram "新模" is completely unseen in our training text
|
|
// Expected behavior: Trigram lookup fails. Model smoothly backs off to the Bigram tier matching only "模" -> "型".
|
|
// Mathematical projection: 1.0 baseline confidence * 0.4 alpha penalty = 0.4 adjusted score.
|
|
c_ucs4_t unseen_trigram_history[] = { 0x65B0, 0x6A21 }; // "新模"
|
|
err = c_NlpLanguageModel_PredictSmooth(&lm, unseen_trigram_history, 2, &next_prediction, &confidence_score);
|
|
|
|
RUN_TEST(err == C_ERR_OK, "Scenario B: Back-off routine successfully processes the unseen 3-gram phrase");
|
|
RUN_TEST(next_prediction == 0x578B, "Scenario B: Lower-order bigram successfully resolves '模' ➔ '型'");
|
|
RUN_TEST(confidence_score == 0.4, "Scenario B: Confidence safely records the 0.4 alpha smoothing decay penalty");
|
|
|
|
c_NlpLanguageModel_Destroy(&lm);
|
|
return C_ERR_OK;
|
|
}
|
|
|
|
int main(int argc, char** argv){
|
|
return test();
|
|
}
|