hermes-sync/sync.sh

#!/bin/bash
set -e

SYNC_DIR="${SYNC_DIR:-$HOME/.hermes-sync}"
HERMES_HOME="${HERMES_HOME:-$HOME/.hermes}"
HOSTNAME=$(hostname)
cd "$SYNC_DIR"

echo "[$(date '+%H:%M:%S')] Sync from $HOSTNAME..."

# ── Step 0: Clean local state_merged.db to avoid git pull conflicts ──────
# The merged db is produced by this script, not tracked, and can conflict
# with remote when pulling. Remove it before fetch+merge.
rm -f "$SYNC_DIR/state_merged.db"

# ── Step 1: Export local state.db (via temp dir to avoid lock & WAL issues) ─
python3 << 'PYEOF'
import sqlite3, os, shutil, tempfile

local_db = os.path.join(os.path.expanduser('~/.hermes'), 'state.db')
export_db = os.path.join(os.path.expanduser('~/.hermes-sync'), f"state_{os.environ.get('HOSTNAME') or __import__('socket').gethostname()}.db")
hostname = os.environ.get('HOSTNAME') or __import__('socket').gethostname()
tmpdir = tempfile.mkdtemp(prefix='hs_exp_')

try:
    # Step 1a: Open with WAL mode and checkpoint (forces WAL -> main db)
    conn = sqlite3.connect(f'file:{local_db}?mode=ro', uri=True)
    jm = conn.execute('PRAGMA journal_mode').fetchone()[0]

    if jm == 'wal':
        # WAL mode: checkpoint first to flush WAL into main db
        conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')
        print(f'WAL checkpointed ({jm})')
    conn.close()

    # Step 1b: Copy all three files (db + -wal + -shm) if they exist
    tmp_db = os.path.join(tmpdir, 'db')
    for suffix in ['', '-wal', '-shm']:
        src = local_db + suffix
        if os.path.exists(src):
            shutil.copy2(src, tmp_db + suffix)

    # Step 1c: Verify (open read-only to confirm it's not corrupted)
    test = sqlite3.connect(f'file:{tmp_db}?mode=ro', uri=True)
    r = test.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
    m = test.execute('SELECT COUNT(*) FROM messages').fetchone()[0]
    test.close()

    shutil.copy2(tmp_db, export_db)
    if os.path.exists(tmp_db + '-wal'):
        shutil.copy2(tmp_db + '-wal', export_db + '-wal')
    if os.path.exists(tmp_db + '-shm'):
        shutil.copy2(tmp_db + '-shm', export_db + '-shm')

    print(f'Exported: {r}s/{m}m (journal={jm})')
finally:
    shutil.rmtree(tmpdir, ignore_errors=True)
PYEOF

# ── Step 2: Git stage ────────────────────────────────────────────────────
git add -A
HAS_LOCAL=false
if ! git diff --cached --quiet || ! git diff --quiet; then
    HAS_LOCAL=true
fi

# ── Step 3: Fetch + merge ─────────────────────────────────────────────────
git fetch origin main

if git rev-parse HEAD >/dev/null 2>&1 && \
   git rev-parse origin/main >/dev/null 2>&1 && \
   ! git merge-base --is-ancestor HEAD origin/main 2>/dev/null; then
    echo "Merging remote..."
    if [ "$HAS_LOCAL" = true ]; then
        git stash push -m "local $(date)" 2>/dev/null || true
        if ! git merge origin/main --no-edit 2>/dev/null; then
            git checkout --ours sync.sh memories/MEMORY.md 2>/dev/null || true
            git add -A
            git commit -m "Auto-resolve $(date)" 2>/dev/null || true
        fi
        if git stash list | grep -q "local "; then
            git stash pop 2>/dev/null || true
            git rebase origin/main 2>/dev/null || {
                git rebase --abort 2>/dev/null || true
                git merge origin/main --no-edit 2>/dev/null || true
            }
        fi
    else
        git merge origin/main --no-edit 2>/dev/null || \
        git merge --ff-only origin/main 2>/dev/null || \
        git reset --hard origin/main
    fi
fi

# ── Step 4: Merge all state_*.db → state_merged.db ──────────────────────
python3 << 'PYEOF'
import sqlite3, os, glob, shutil, tempfile

sync_dir = os.path.expanduser('~/.hermes-sync')
merged_path = os.path.join(sync_dir, 'state_merged.db')
db_files = sorted(glob.glob(os.path.join(sync_dir, 'state_*.db')))
db_files = [f for f in db_files if not f.endswith('_merged.db')]
print(f'Merging {len(db_files)} DBs')

tmpdir = tempfile.mkdtemp(prefix='hs_merge_')
tmp_merged = os.path.join(tmpdir, 'merged.db')

try:
    # Create merged DB (DELETE journal avoids WAL complications during merge)
    conn = sqlite3.connect(tmp_merged)
    conn.execute('PRAGMA journal_mode=DELETE')
    conn.execute('PRAGMA locking_mode=NORMAL')
    conn.execute('PRAGMA synchronous=FULL')

    # Use one reference DB to get exact schema (handles FTS, etc.)
    ref_db = db_files[0]
    ref_copy = os.path.join(tmpdir, 'ref.db')
    shutil.copy2(ref_db, ref_copy)
    if os.path.exists(ref_db + '-wal'):
        shutil.copy2(ref_db + '-wal', ref_copy + '-wal')
    if os.path.exists(ref_db + '-shm'):
        shutil.copy2(ref_db + '-shm', ref_copy + '-shm')

    ref = sqlite3.connect(f'file:{ref_copy}?mode=ro', uri=True)
    for line in ref.iterdump():
        # Skip FTS virtual table data rows (they are derived, not real data)
        if line.startswith('INSERT INTO messages_fts') or \
           line.startswith('DELETE FROM messages_fts'):
            continue
        try:
            conn.execute(line)
        except Exception as e:
            pass  # Ignore schema errors from partial DDL
    ref.close()
    os.remove(ref_copy)
    if os.path.exists(ref_copy + '-wal'):
        os.remove(ref_copy + '-wal')

    # Copy sessions (INSERT OR REPLACE to dedupe by PK)
    for db_file in db_files:
        name = os.path.basename(db_file)

        # Copy to temp with WAL files
        tmp_copy = os.path.join(tmpdir, name)
        shutil.copy2(db_file, tmp_copy)
        if os.path.exists(db_file + '-wal'):
            shutil.copy2(db_file + '-wal', tmp_copy + '-wal')
        if os.path.exists(db_file + '-shm'):
            shutil.copy2(db_file + '-shm', tmp_copy + '-shm')

        src = sqlite3.connect(f'file:{tmp_copy}?mode=ro', uri=True)

        s_cnt = src.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
        m_cnt = src.execute('SELECT COUNT(*) FROM messages').fetchone()[0]
        jm = src.execute('PRAGMA journal_mode').fetchone()[0]
        print(f'  {name}: {s_cnt}s/{m_cnt}m journal={jm}')

        sess_rows = src.execute('SELECT * FROM sessions').fetchall()
        sess_cols = len(src.execute('PRAGMA table_info(sessions)').fetchall())
        for row in sess_rows:
            conn.execute(f'INSERT OR REPLACE INTO sessions VALUES ({",".join(["?"]*sess_cols)})', row)

        msg_rows = src.execute('SELECT * FROM messages').fetchall()
        msg_cols = len(src.execute('PRAGMA table_info(messages)').fetchall())
        for row in msg_rows:
            conn.execute(f'INSERT OR IGNORE INTO messages VALUES ({",".join(["?"]*msg_cols)})', row)

        src.close()
        os.remove(tmp_copy)
        for suf in ['-wal', '-shm']:
            if os.path.exists(tmp_copy + suf):
                os.remove(tmp_copy + suf)

    conn.commit()
    conn.close()

    if os.path.exists(merged_path):
        os.remove(merged_path)
    shutil.copy2(tmp_merged, merged_path)
    # Ensure no WAL on merged (merge output should be clean DELETE)
    for suf in ['-wal', '-shm']:
        if os.path.exists(merged_path + suf):
            os.remove(merged_path + suf)
    print(f'Merged: {os.path.getsize(merged_path)/1024:.0f} KB')

finally:
    shutil.rmtree(tmpdir, ignore_errors=True)
PYEOF

# ── Step 5: Stage merged DB (it was just created in Step 4) ─────────────
git add state_merged.db 2>/dev/null || true

# ── Step 6: Push ─────────────────────────────────────────────────────────
if [ "$HAS_LOCAL" = true ]; then
    git commit -m "Sync $(date '+%Y-%m-%d %H:%M')" 2>/dev/null || true
    if ! git push origin main 2>&1; then
        echo "Push rejected, pulling..."
        git pull origin main --no-edit 2>/dev/null || true
        git push origin main 2>&1 || echo "Push failed"
    fi
else
    echo "No local changes"
fi

# ── Step 7: Restore merged state to local hermes ─────────────────────────
python3 << 'PYEOF'
import sqlite3, os, shutil, tempfile

hermes_home = os.path.expanduser('~/.hermes')
merged_path = os.path.join(os.path.expanduser('~/.hermes-sync'), 'state_merged.db')
local_db = os.path.join(hermes_home, 'state.db')

if not os.path.exists(merged_path):
    print('No merged DB, skipping restore')
else:
    tmpdir = tempfile.mkdtemp(prefix='hs_rest_')
    try:
        # Step 7a: Ensure merged db is fully checkpointed (DELETE journal mode)
        merge_conn = sqlite3.connect(f'file:{merged_path}?mode=ro', uri=True)
        merge_jm = merge_conn.execute('PRAGMA journal_mode').fetchone()[0]
        if merge_jm == 'wal':
            merge_conn.execute('PRAGMA wal_checkpoint(TRUNCATE)')
            print(f'Restored merged DB had WAL, checkpointed ({merge_jm})')
        merge_conn.close()

        tmp_db = os.path.join(tmpdir, 'db')
        shutil.copy2(merged_path, tmp_db)
        # Also copy -wal/-shm if present
        for suf in ['-wal', '-shm']:
            if os.path.exists(merged_path + suf):
                shutil.copy2(merged_path + suf, tmp_db + suf)

        # Step 7b: Verify before overwriting
        test = sqlite3.connect(f'file:{tmp_db}?mode=ro', uri=True)
        r = test.execute('SELECT COUNT(*) FROM sessions').fetchone()[0]
        m = test.execute('SELECT COUNT(*) FROM messages').fetchone()[0]
        test.execute('PRAGMA integrity_check')  # verify
        test.close()

        shutil.copy2(local_db, local_db + '.bak')
        shutil.copy2(tmp_db, local_db)
        # Also copy WAL files to local
        for suf in ['-wal', '-shm']:
            if os.path.exists(tmp_db + suf):
                shutil.copy2(tmp_db + suf, local_db + suf)
            elif os.path.exists(local_db + suf):
                os.remove(local_db + suf)
        os.remove(local_db + '.bak')
        print(f'Restored: {r}s/{m}m')
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True)
PYEOF

# ── Step 8: Sync memories + skills (additive) ────────────────────────────
cp "$SYNC_DIR/memories/MEMORY.md" "$HERMES_HOME/memories/MEMORY.md" 2>/dev/null || true
if [ -d "$SYNC_DIR/skills" ]; then
    mkdir -p "$HERMES_HOME/skills"
    rsync -a --ignore-existing "$SYNC_DIR/skills/" "$HERMES_HOME/skills/" 2>/dev/null || \
        cp -rn "$SYNC_DIR/skills/"* "$HERMES_HOME/skills/" 2>/dev/null || true
fi

echo "[$(date '+%H:%M:%S')] Done"