Refactor: Optimize DB performance, reduce repetitions, and add @-ping protection

- Optimized Markov chain storage: Switched from full rewrites to incremental DB updates.
- Improved AI creativity: Reduced repetitions using presence/frequency penalties and prompt tuning.
- Increased Markov randomness: Lowered order to 1 and enabled learning of special characters/emojis.
- Added @-ping protection: Automatically strips '@' symbols from AI and Markov responses.
- Enhanced robustness: Added startup token checks, directory auto-creation, and admin/bot-info caching.

src/markov.ts

@@ -14,7 +14,7 @@ export class MarkovChain {
   private order: number;
   private chain: ChainData;
 
-  constructor(order: number = 2) {
+  constructor(order: number = 1) {
     this.order = order;
     this.chain = {
       transitions: new Map(),
@@ -23,35 +23,29 @@ export class MarkovChain {
   }
 
   /**
-   * Tokenize text into words, preserving sentence boundaries
+   * Tokenize text into words.
+   * Keeps special characters and emojis as requested.
    */
   private tokenize(text: string): string[][] {
-    const sentences: string[][] = [];
+    // We treat the whole message as one sequence to keep it simple and creative
+    const words = text
+      .trim()
+      .split(/\s+/)
+      .filter(w => w.length > 0);
 
-    // Split into sentences (rough but works for most cases)
-    const sentencePatterns = text.split(/[.!?]+/);
-
-    for (const sentence of sentencePatterns) {
-      const words = sentence
-        .trim()
-        .toLowerCase()
-        .replace(/[^\wäöüß\s]/g, '') // Keep German characters
-        .split(/\s+/)
-        .filter(w => w.length > 0);
-
-      if (words.length > 0) {
-        sentences.push(words);
-      }
-    }
-
-    return sentences;
+    return words.length > 0 ? [words] : [];
   }
 
   /**
-   * Learn from a text, adding it to the chain
+   * Learn from a text, adding it to the chain.
+   * Returns the new transitions and starts for efficient DB updates.
    */
-  learn(text: string): void {
+  learn(text: string): { transitions: Array<{ key: string; next: string }>; starts: string[] } {
     const sentences = this.tokenize(text);
+    const learned = {
+      transitions: [] as Array<{ key: string; next: string }>,
+      starts: [] as string[],
+    };
 
     for (const words of sentences) {
       if (words.length < this.order + 1) continue;
@@ -59,6 +53,7 @@ export class MarkovChain {
       // Mark sentence start
       const startKey = words.slice(0, this.order).join(' ');
       this.chain.starts.set(startKey, (this.chain.starts.get(startKey) || 0) + 1);
+      learned.starts.push(startKey);
 
       // Build transitions
       for (let i = 0; i < words.length - this.order; i++) {
@@ -71,8 +66,11 @@ export class MarkovChain {
 
         const transitions = this.chain.transitions.get(key)!;
         transitions.set(next, (transitions.get(next) || 0) + 1);
+        learned.transitions.push({ key, next });
       }
     }
+
+    return learned;
   }
 
   /**