Merge branch 'main' of https://d.momix.cc/Mixer/hermes-sync

.gitignore

@@ -7,3 +7,4 @@ __pycache__/
 *.db-shm
 *.db-journal
 !state_*.db
+state_merged.db

memories/MEMORY.md

@@ -26,4 +26,33 @@ RackNerd VPS (New Year Special):
 - Username: root
 - Root Password: Zg58Mb73C4ueHFYhq3
 - 已安装 hermes agent
-- 问题: Telegram 对话失联无响应
+- 问题: Telegram 对话失联无响应
+§
+## Hermes Sync 架构 (2026-04-14 建成)
+
+Gitea: https://d.momix.cc/Mixer/hermes-sync
+
+三台机器：
+- 本地 Linux (10-40-29-186): ~/.hermes-sync (symlink to ~/hermes-sync-tmp), ~/.hermes
+- VPS RackNerd (soe219028): ~/.hermes-sync, ~/.hermes
+- MacOS (MixdeMac-mini.local): /Users/moore/.hermes-sync, /Users/moore/.hermes
+
+同步内容：memories/MEMORY.md, skills/, state.db（每机器一个 state_<hostname>.db）
+不同步：config.yaml, .env, auth.json 等配置
+
+sync.sh 已稳定：217行，无checkpoint，直接copy到temp dir再处理，避免SQLite WAL锁冲突
+cron: 每2小时，VPS和MacOS用绝对路径 /Users/moore/.hermes-sync/sync.sh
+本地: 软链 ~/hermes-sync-tmp -> ~/hermes-sync
+
+MacOS hostname: MixdeMac-mini.local
+
+## macOS Hermes 路径
+- 数据目录: /Users/moore/.hermes/state.db
+- Hermes CLI: /Users/moore/.local/bin/hermes
+- 无systemd，用cron管理后台进程
+
+## SQLite 多机同步关键经验
+- WAL模式的db不能直接copy，会损坏。用temp dir中转。
+- 不能对WAL模式的db做checkpoint（会造成malformed image），直接shutil.copy2即可，WAL会在下次连接时自动attach。
+- merge时sessions用INSERT OR REPLACE（去重），messages用INSERT OR IGNORE（避免重复插入）
+- 列数用PRAGMA table_info动态获取，不硬编码

sync.sh

@@ -8,24 +8,32 @@ cd "$SYNC_DIR"
 
 echo "[$(date '+%H:%M:%S')] Sync from $HOSTNAME..."
 
-# ── Step 1: Export local state.db → state_<hostname>.db ──────────────────
+# ── Step 0: Clean local state_merged.db to avoid git pull conflicts ──────
+# The merged db is produced by this script, not tracked, and can conflict
+# with remote when pulling. Remove it before fetch+merge.
+rm -f "$SYNC_DIR/state_merged.db"
+
+# ── Step 1: Export local state.db (via temp dir to avoid lock) ────────────
 python3 << PYEOF
-import sqlite3, os, shutil
+import sqlite3, os, shutil, tempfile
 
-local_db = os.path.join(os.environ['HERMES_HOME'], 'state.db')
+local_db = os.path.join(os.path.expanduser('~/.hermes'), 'state.db')
-export_db = os.path.join(os.environ['SYNC_DIR'], f"state_{os.environ['HOSTNAME']}.db")
+export_db = os.path.join(os.path.expanduser('~/.hermes-sync'), f"state_{os.environ.get('HOSTNAME') or __import__('socket').gethostname()}.db")
+tmpdir = tempfile.mkdtemp(prefix='hs_exp_')
 
-# Checkpoint WAL
 try:
-    conn = sqlite3.connect(local_db)
+    tmp_db = os.path.join(tmpdir, 'db')
-    conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')
+    shutil.copy2(local_db, tmp_db)
-    conn.execute('PRAGMA optimize')
-    conn.close()
-except Exception as e:
-    print(f'Checkpoint: {e}', file=__import__('sys').stderr)
 
-shutil.copy2(local_db, export_db)
+    test = sqlite3.connect(tmp_db)
-print(f'Exported: {export_db}')
+    r = test.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
+    m = test.execute('SELECT COUNT(*) FROM messages').fetchone()[0]
+    test.close()
+
+    shutil.copy2(tmp_db, export_db)
+    print(f'Exported: {r}s/{m}m')
+finally:
+    shutil.rmtree(tmpdir, ignore_errors=True)
 PYEOF
 
 # ── Step 2: Git stage ────────────────────────────────────────────────────
@@ -65,42 +73,26 @@ fi
 
 # ── Step 4: Merge all state_*.db → state_merged.db ──────────────────────
 python3 << 'PYEOF'
-import sqlite3, os, glob, shutil
+import sqlite3, os, glob, shutil, tempfile
 
-sync_dir = os.environ['SYNC_DIR']
+sync_dir = os.path.expanduser('~/.hermes-sync')
-hermes_home = os.environ['HERMES_HOME']
 merged_path = os.path.join(sync_dir, 'state_merged.db')
 db_files = sorted(glob.glob(os.path.join(sync_dir, 'state_*.db')))
 db_files = [f for f in db_files if not f.endswith('_merged.db')]
-print(f'DBs to merge: {[os.path.basename(f) for f in db_files]}')
+print(f'Merging {len(db_files)} DBs')
 
-if os.path.exists(merged_path):
+tmpdir = tempfile.mkdtemp(prefix='hs_merge_')
-    os.remove(merged_path)
+tmp_merged = os.path.join(tmpdir, 'merged.db')
 
-def get_schema(conn, table):
+try:
-    cols = conn.execute(f'PRAGMA table_info("{table}")').fetchall()
+    # Create merged DB with proper schema
-    return [c[1] for c in cols]  # column names
+    conn = sqlite3.connect(tmp_merged)
+    conn.execute('PRAGMA journal_mode=DELETE')
+    conn.execute('PRAGMA locking_mode=NORMAL')
+    conn.execute('PRAGMA synchronous=FULL')
 
-def copy_table(src_conn, dst_conn, table):
+    conn.execute('''
-    cols = get_schema(src_conn, table)
+    CREATE TABLE sessions (
-    placeholders = ','.join(['?'] * len(cols))
-    col_names = ','.join(f'"{c}"' for c in cols)
-    
-    rows = src_conn.execute(f'SELECT {col_names} FROM "{table}"').fetchall()
-    for row in rows:
-        dst_conn.execute(f'INSERT OR REPLACE INTO "{table}" ({col_names}) VALUES ({placeholders})', row)
-    return len(rows)
-
-# Open merged DB and rebuild from scratch
-conn_merged = sqlite3.connect(merged_path)
-conn_merged.execute('PRAGMA journal_mode=WAL')
-conn_merged.execute('PRAGMA synchronous=NORMAL')
-
-# We need at least one source to initialize schema
-if not db_files:
-    print('No state DBs found, creating empty merged DB')
-    conn_merged.execute('''
-    CREATE TABLE IF NOT EXISTS sessions (
         id TEXT PRIMARY KEY, source TEXT NOT NULL, user_id TEXT,
         model TEXT, model_config TEXT, system_prompt TEXT,
         parent_session_id TEXT, started_at REAL, ended_at REAL,
@@ -112,8 +104,9 @@ if not db_files:
         estimated_cost_usd REAL, actual_cost_usd REAL,
         cost_status TEXT, cost_source TEXT, pricing_version TEXT, title TEXT
     )''')
-    conn_merged.execute('''
+
-    CREATE TABLE IF NOT EXISTS messages (
+    conn.execute('''
+    CREATE TABLE messages (
         id INTEGER PRIMARY KEY,
         session_id TEXT NOT NULL, role TEXT NOT NULL, content TEXT,
         tool_call_id TEXT, tool_calls TEXT, tool_name TEXT,
@@ -121,43 +114,56 @@ if not db_files:
         finish_reason TEXT, reasoning TEXT, reasoning_details TEXT,
         codex_reasoning_items TEXT
     )''')
-    conn_merged.commit()
-else:
-    first_db = db_files[0]
-    src_conn = sqlite3.connect(first_db)
-    src_conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')
-    
-    for table in ['sessions', 'messages']:
-        cols = get_schema(src_conn, table)
-        col_defs = []
-        for c in cols:
-            if c == 'id' and table == 'messages':
-                col_defs.append('id INTEGER PRIMARY KEY')
-            elif c == 'id':
-                col_defs.append(f'{c} TEXT PRIMARY KEY')
-            else:
-                col_defs.append(f'{c} TEXT')
-        
-        conn_merged.execute(f'DROP TABLE IF EXISTS "{table}"')
-        conn_merged.execute(f'CREATE TABLE "{table}" ({", ".join(col_defs)})')
-        
-        placeholders = ','.join(['?'] * len(cols))
-        col_names = ','.join(f'"{c}"' for c in cols)
-        rows = src_conn.execute(f'SELECT {col_names} FROM "{table}"').fetchall()
-        for row in rows:
-            conn_merged.execute(f'INSERT OR REPLACE INTO "{table}" ({col_names}) VALUES ({placeholders})', row)
-        print(f'  {os.path.basename(first_db)}.{table}: {len(rows)} rows')
 
-    src_conn.close()
+    conn.execute('CREATE INDEX idx_msg_session ON messages(session_id)')
+    conn.execute('CREATE INDEX idx_msg_ts ON messages(timestamp)')
 
-conn_merged.execute('PRAGMA wal_checkpoint(TRUNCATE)')
+    for db_file in db_files:
-conn_merged.commit()
+        name = os.path.basename(db_file)
-conn_merged.close()
+        # Copy to temp file first (avoids WAL lock issues with open connections)
+        tmp_copy = os.path.join(tmpdir, name)
+        shutil.copy2(db_file, tmp_copy)
+        # Also copy WAL if exists
+        if os.path.exists(db_file + '-wal'):
+            shutil.copy2(db_file + '-wal', tmp_copy + '-wal')
 
-print(f'Merged: {merged_path} ({os.path.getsize(merged_path)/1024:.0f} KB)')
+        src = sqlite3.connect(tmp_copy)
+
+        s_cnt = src.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
+        m_cnt = src.execute('SELECT COUNT(*) FROM messages').fetchone()[0]
+        print(f'  {name}: {s_cnt}s/{m_cnt}m')
+
+        sess_rows = src.execute('SELECT * FROM sessions').fetchall()
+        sess_cols = len(src.execute('PRAGMA table_info(sessions)').fetchall())
+        for row in sess_rows:
+            conn.execute(f'INSERT OR REPLACE INTO sessions VALUES ({",".join(["?"]*sess_cols)})', row)
+
+        msg_rows = src.execute('SELECT * FROM messages').fetchall()
+        msg_cols = len(src.execute('PRAGMA table_info(messages)').fetchall())
+        for row in msg_rows:
+            conn.execute(f'INSERT OR IGNORE INTO messages VALUES ({",".join(["?"]*msg_cols)})', row)
+
+        src.close()
+        os.remove(tmp_copy)
+        if os.path.exists(tmp_copy + '-wal'):
+            os.remove(tmp_copy + '-wal')
+
+    conn.commit()
+    conn.close()
+
+    if os.path.exists(merged_path):
+        os.remove(merged_path)
+    shutil.copy2(tmp_merged, merged_path)
+    print(f'Merged: {os.path.getsize(merged_path)/1024:.0f} KB')
+
+finally:
+    shutil.rmtree(tmpdir, ignore_errors=True)
 PYEOF
 
-# ── Step 5: Push ─────────────────────────────────────────────────────────
+# ── Step 5: Stage merged DB (it was just created in Step 4) ─────────────
+git add state_merged.db 2>/dev/null || true
+
+# ── Step 6: Push ─────────────────────────────────────────────────────────
 if [ "$HAS_LOCAL" = true ]; then
     git commit -m "Sync $(date '+%Y-%m-%d %H:%M')" 2>/dev/null || true
     if ! git push origin main 2>&1; then
@@ -169,41 +175,36 @@ else
     echo "No local changes"
 fi
 
-# ── Step 6: Restore merged state to local hermes ─────────────────────────
+# ── Step 7: Restore merged state to local hermes ─────────────────────────
 python3 << 'PYEOF'
-import sqlite3, os, shutil
+import sqlite3, os, shutil, tempfile
 
-sync_dir = os.environ['SYNC_DIR']
+hermes_home = os.path.expanduser('~/.hermes')
-hermes_home = os.environ['HERMES_HOME']
+merged_path = os.path.join(os.path.expanduser('~/.hermes-sync'), 'state_merged.db')
-merged_path = os.path.join(sync_dir, 'state_merged.db')
 local_db = os.path.join(hermes_home, 'state.db')
 
 if not os.path.exists(merged_path):
     print('No merged DB, skipping restore')
 else:
-    # Checkpoint + close local
+    tmpdir = tempfile.mkdtemp(prefix='hs_rest_')
     try:
-        conn = sqlite3.connect(local_db)
+        tmp_db = os.path.join(tmpdir, 'db')
-        conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')
+        shutil.copy2(merged_path, tmp_db)
-        conn.execute('PRAGMA optimize')
+
-        conn.close()
+        test = sqlite3.connect(tmp_db)
-    except Exception as e:
+        r = test.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
-        print(f'Local checkpoint: {e}')
+        m = test.execute('SELECT COUNT(*) FROM messages').fetchone()[0]
-    
+        test.close()
-    # Backup
+
-    shutil.copy2(local_db, local_db + '.pre_sync_bak')
+        shutil.copy2(local_db, local_db + '.bak')
-    
+        shutil.copy2(tmp_db, local_db)
-    # Replace with merged
+        os.remove(local_db + '.bak')
-    shutil.copy2(merged_path, local_db)
+        print(f'Restored: {r}s/{m}m')
-    print(f'Restored merged state to {local_db}')
+    finally:
-    
+        shutil.rmtree(tmpdir, ignore_errors=True)
-    # Remove backup after successful restore
-    bak = local_db + '.pre_sync_bak'
-    if os.path.exists(bak):
-        os.remove(bak)
 PYEOF
 
-# ── Step 7: Sync memories + skills (additive) ────────────────────────────
+# ── Step 8: Sync memories + skills (additive) ────────────────────────────
 cp "$SYNC_DIR/memories/MEMORY.md" "$HERMES_HOME/memories/MEMORY.md" 2>/dev/null || true
 if [ -d "$SYNC_DIR/skills" ]; then
     mkdir -p "$HERMES_HOME/skills"