同步当前架构和数据 (2026-06-11)

- 训练脚本: train_v2.py + llm_chat_improved.py
- API生成: api_correction.py
- 高温多样化: regenerate_all_high_temp.py
- 每日数据生成: daily_data_gen.py
- 数据: xiaoli_train_v2.json(40776条) + vocab_v2.json
- 蒸馏问题: exam_distill_questions等
- Web服务: web_server.py + chat.html
- 新增 .gitignore 排除旧版文件

llm_chat_improved.py

@@ -35,6 +35,7 @@ class CharTokenizer:
         self.char_to_id = self.special_tokens.copy()
         self.id_to_char = {v: k for k, v in self.char_to_id.items()}
         self.next_id = len(self.special_tokens)
+        self._sorted_vocab_cache = None  # encode缓存用
     
     def build_vocab(self, texts: List[str], min_freq: int = 1) -> Dict[str, int]:
         """从文本构建词汇表"""
@@ -53,6 +54,13 @@ class CharTokenizer:
                 self.id_to_char[self.next_id] = char
                 self.next_id += 1
         
+        # 清除encode缓存（vocab已变化）
+        self._sorted_vocab_cache = None
+        # 清除多字符token缓存
+        for attr in ['_multi_trie', '_multi_keys', '_single_map']:
+            if hasattr(self, attr):
+                delattr(self, attr)
+        
         return self.char_to_id
     
     def _split_chars(self, text: str) -> List[str]:
@@ -71,18 +79,38 @@ class CharTokenizer:
         return result
     
     def encode(self, text: str) -> List[int]:
-        """将文本编码为ID序列"""
+        """将文本编码为ID序列 - 高效版"""
         if not text:
             return [self.special_tokens['<UNK>']]
         
-        chars = self._split_chars(text)
         ids = []
-        for char in chars:
-            if char in self.char_to_id:
-                ids.append(self.char_to_id[char])
-            else:
-                # 未知字符用 UNK token
-                ids.append(self.special_tokens['<UNK>'])
+        i = 0
+        n = len(text)
+        
+        # 构建多字符token的前缀映射（仅一次）
+        if not hasattr(self, '_multi_trie'):
+            self._multi_trie = {}
+            for token in self.char_to_id:
+                if len(token) > 1:
+                    self._multi_trie[token] = self.char_to_id[token]
+            # 按长度降序排序，优先匹配长token
+            self._multi_keys = sorted(self._multi_trie.keys(), key=len, reverse=True)
+            self._single_map = {c: tid for c, tid in self.char_to_id.items() if len(c) == 1}
+        
+        while i < n:
+            matched = False
+            # 优先匹配多字符token（特殊标记）
+            for token in self._multi_keys:
+                if text.startswith(token, i):
+                    ids.append(self._multi_trie[token])
+                    i += len(token)
+                    matched = True
+                    break
+            if not matched:
+                # 单字符直接用dict查
+                c = text[i]
+                ids.append(self._single_map.get(c, self.special_tokens['<UNK>']))
+                i += 1
         
         return ids
     
@@ -92,8 +120,8 @@ class CharTokenizer:
         for tid in ids:
             if tid in self.id_to_char:
                 char = self.id_to_char[tid]
-                if char not in self.special_tokens:
-                    chars.append(char)
+                # 保留特殊token以便后续处理，但这里先不过滤
+                chars.append(char)
         return ''.join(chars)
 
 
@@ -512,7 +540,7 @@ class LLMModel(nn.Module):
 class ConversationDataset(Dataset):
     """对话数据集"""
     
-    def __init__(self, conversations: List[Tuple[str, str]], vocab: dict, max_length: int = 128):
+    def __init__(self, conversations: List[Tuple[str, str]], vocab: dict, max_length: int = 256):
         self.conversations = conversations
         self.vocab = vocab
         self.max_length = max_length
@@ -582,7 +610,7 @@ class LLMTrainer:
         self.model.train()
         total_loss = 0
         accumulation_steps = config.get('gradient_accumulation_steps', 1) if config else 1
-        log_interval = max(1, len(dataloader) // 10)  # 每10%打印一次
+        log_interval = max(1, len(dataloader) // 100)  # 每1%打印一次
         
         for batch_idx, batch in enumerate(dataloader):
             input_ids = batch['input_ids'].to(self.device, non_blocking=True)
@@ -637,6 +665,27 @@ class LLMTrainer:
         self.scheduler.step()
         return total_loss / len(dataloader)
     
+    def evaluate(self, dataloader: DataLoader, config: dict = None):
+        """在验证集上计算loss（无梯度，不更新权重）"""
+        self.model.eval()
+        total_loss = 0
+        accumulation_steps = config.get('gradient_accumulation_steps', 1) if config else 1
+        
+        with torch.no_grad():
+            for batch_idx, batch in enumerate(dataloader):
+                input_ids = batch['input_ids'].to(self.device, non_blocking=True)
+                target_ids = batch['target_ids'].to(self.device, non_blocking=True)
+                
+                mask = self.model.generate_mask(input_ids.size(1)).to(self.device)
+                outputs = self.model(input_ids, mask)
+                outputs = outputs.reshape(-1, outputs.size(-1))
+                tgt = target_ids.reshape(-1)
+                
+                loss = self.criterion(outputs, tgt)
+                total_loss += loss.item()
+        
+        return total_loss / len(dataloader)
+    
     def save_model(self, filepath: str, vocab: dict, epoch: int = None):
         save_dict = {
             'model_state_dict': self.model.state_dict(),
@@ -677,7 +726,7 @@ class LLMTrainer:
 class LLMChat:
     """LLM对话"""
     
-    def __init__(self, model: nn.Module, vocab: dict, device: str = 'cpu', max_length: int = 128,
+    def __init__(self, model: nn.Module, vocab: dict, device: str = 'cpu', max_length: int = 256,
                  show_thinking: bool = True):
         self.model = model.to(device)
         self.model.eval()
@@ -692,14 +741,30 @@ class LLMChat:
         # 同步全局tokenizer的vocab（关键！）
         tokenizer_instance.char_to_id = vocab
         tokenizer_instance.id_to_char = {v: k for k, v in vocab.items()}
+        # 清除encode缓存，避免用旧的cache导致错位
+        tokenizer_instance._sorted_vocab_cache = None
+        for attr in ['_multi_trie', '_multi_keys', '_single_map']:
+            if hasattr(tokenizer_instance, attr):
+                delattr(tokenizer_instance, attr)
     
     def generate_response(self, question: str, max_length: int = 100, temperature: float = 0.8,
-                         include_thinking: bool = False, use_beam_search: bool = False):
-        """生成回答"""
+                         include_thinking: bool = False, use_beam_search: bool = False,
+                         history: list = None):
+        """生成回答
+        history: 可选，历史对话列表 [(user_q, assistant_a), ...]
+        """
         self.model.eval()
         
-        # 使用与训练数据一致的格式: <SOS>问题<SEP><START>
-        input_text = f"<SOS>{question}<SEP><START>"
+        # 构建输入（支持多轮上下文）
+        if history:
+            parts = []
+            for hq, ha in history:
+                parts.append(hq)
+                parts.append(ha)
+            parts.append(question)
+            input_text = f"<SOS>{'<SEP>'.join(parts)}<SEP><START>"
+        else:
+            input_text = f"<SOS>{question}<SEP><START>"
         
         input_ids = tokenizer_instance.encode(input_text)