chore(ops): add nightly backup verification scripts

.github/workflows/deploy.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-go@v6
         with:
-          go-version: '1.23'
+          go-version-file: go.mod
       - name: Run tests
         run: go test ./...
       - name: Deploy to server

.gitignore

@@ -114,3 +114,6 @@ ROADMAP.md
 
 *.bak
 terminalshop.prod.*
+
+# Backup verification config (real values; example file is committed)
+scripts/backups/config.env

api/handlers/cart.go

@@ -23,13 +23,14 @@ import (
 )
 
 type CartHandler struct {
-	stripeKey string
+	stripeKey     string
-	appURL    string
+	appURL        string
+	maxOrderCents int
 }
 
 // NewCartHandler creates a new cart handler with the Stripe secret key
-func NewCartHandler(stripeSecretKey, appURL string) *CartHandler {
+func NewCartHandler(stripeSecretKey, appURL string, maxOrderCents int) *CartHandler {
-	return &CartHandler{stripeKey: stripeSecretKey, appURL: appURL}
+	return &CartHandler{stripeKey: stripeSecretKey, appURL: appURL, maxOrderCents: maxOrderCents}
 }
 
 // getOrCreateCart finds the users cart or creates one.
@@ -354,6 +355,17 @@ func (h *CartHandler) ConvertCart(w http.ResponseWriter, r *http.Request) {
 
 	total := subtotal + cart.ShippingCost
 
+	if h.maxOrderCents > 0 && total > h.maxOrderCents {
+		middleware.RecordCartConversion("validation_over_limit")
+		utils.RespondError(w, http.StatusBadRequest, "CART_OVER_LIMIT",
+			fmt.Sprintf("order total must be at most $%.2f", float64(h.maxOrderCents)/100),
+			map[string]interface{}{
+				"total_cents": total,
+				"limit_cents": h.maxOrderCents,
+			})
+		return
+	}
+
 	if total < 50 {
 		middleware.RecordCartConversion("validation_min_amount")
 		utils.RespondError(w, http.StatusBadRequest, "VALIDATION_ERROR", "order total must be at least $0.50", nil)

api/handlers/cart_test.go

@@ -65,7 +65,7 @@ func TestGetCartEmpty(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	req := authRequest("GET", "/api/v1/cart", nil, user.ID)
 	w := httptest.NewRecorder()
@@ -104,7 +104,7 @@ func TestSetItemAndGetCart(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item (CoffeeID=1, Quantity=2)
 	body, _ := json.Marshal(map[string]interface{}{
@@ -160,7 +160,7 @@ func TestSetItemUpdateQuantity(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 1, "quantity": 2})
@@ -205,7 +205,7 @@ func TestSetItemRemove(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 1, "quantity": 3})
@@ -244,7 +244,7 @@ func TestSetItemInvalidProduct(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 999, "quantity": 1})
 	req := authRequest("PUT", "/api/v1/cart/item", body, user.ID)
@@ -261,7 +261,7 @@ func TestSetItemMissingCoffeeID(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"quantity": 1})
 	req := authRequest("PUT", "/api/v1/cart/item", body, user.ID)
@@ -278,7 +278,7 @@ func TestClearCart(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add two items
 	for _, coffeeID := range []uint{1, 2} {
@@ -334,7 +334,7 @@ func TestSetAddress(t *testing.T) {
 	}
 	db.Create(&addr)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"address_id": addr.ID})
 	req := authRequest("PUT", "/api/v1/cart/address", body, user.ID)
@@ -388,7 +388,7 @@ func TestSetAddressWrongUser(t *testing.T) {
 	}
 	db.Create(&addr)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"address_id": addr.ID})
 	req := authRequest("PUT", "/api/v1/cart/address", body, user.ID)
@@ -416,7 +416,7 @@ func TestSetCard(t *testing.T) {
 	}
 	db.Create(&card)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"card_id": card.ID})
 	req := authRequest("PUT", "/api/v1/cart/card", body, user.ID)
@@ -468,7 +468,7 @@ func TestSetCardWrongUser(t *testing.T) {
 	}
 	db.Create(&card)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"card_id": card.ID})
 	req := authRequest("PUT", "/api/v1/cart/card", body, user.ID)
@@ -485,7 +485,7 @@ func TestConvertCartMissingAddress(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item but no address or card
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 1, "quantity": 1})
@@ -508,7 +508,7 @@ func TestConvertCartMissingCard(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 	db := database.GetDB()
 
 	// Add item and address
@@ -539,7 +539,7 @@ func TestConvertCartEmpty(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Empty cart convert
 	req := authRequest("POST", "/api/v1/cart/convert", nil, user.ID)
@@ -556,7 +556,7 @@ func TestMultipleItems(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add three different items
 	for i := uint(1); i <= 3; i++ {
@@ -605,7 +605,7 @@ func TestCartIsolation(t *testing.T) {
 	}
 	db.Create(&user2)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 	r := chi.NewRouter()
 	r.Put("/cart/item", handler.SetItem)
 	r.Get("/cart", handler.GetCart)
@@ -664,3 +664,103 @@ func TestCartIsolation(t *testing.T) {
 		t.Errorf("User 2 expected coffee_id 2, got %d", resp.Data.Cart.Items[0].CoffeeID)
 	}
 }
+
+func TestConvertCartLimit(t *testing.T) {
+	cases := []struct {
+		name          string
+		capCents      int
+		quantity      int
+		expectOverCap bool
+	}{
+		{"over cap rejects", 20000, 41, true},   // $205
+		{"at cap accepts", 20000, 40, false},    // $200 boundary
+		{"under cap accepts", 20000, 39, false}, // $195
+		{"zero cap disables", 0, 1000, false},   // explicit off-switch
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			testDB, user := setupCartTestDB(t)
+			defer func() { _ = os.Remove(testDB) }()
+			defer database.ResetForTesting()
+
+			db := database.GetDB()
+			user.StripeCustomerID = "cus_test_overlimit"
+			if err := db.Save(&user).Error; err != nil {
+				t.Fatalf("saved user: %v", err)
+			}
+
+			addr := models.Address{
+				UserID: user.ID, Name: "Test", Street: "1 Main St",
+				City: "PDX", State: "OR", Zip: "97201", Country: "US",
+			}
+			if err := db.Create(&addr).Error; err != nil {
+				t.Fatalf("seed address: %v", err)
+			}
+
+			card := models.Card{
+				UserID: user.ID, StripePaymentID: "pm_test_overlimit",
+				Last4: "4242", Brand: "Visa", ExpMonth: 12, ExpYear: 2030,
+			}
+			if err := db.Create(&card).Error; err != nil {
+				t.Fatalf("seed card: %v", err)
+			}
+
+			handler := NewCartHandler("", "", tc.capCents)
+
+			body, _ := json.Marshal(map[string]interface{}{"coffee_id": 4, "quantity": tc.quantity})
+			req := authRequest("PUT", "/api/v1/cart/item", body, user.ID)
+			w := httptest.NewRecorder()
+			handler.SetItem(w, req)
+			if w.Code != http.StatusOK {
+				t.Fatalf("SetItem: %d %s", w.Code, w.Body.String())
+			}
+
+			// set address
+			body, _ = json.Marshal(map[string]interface{}{"address_id": addr.ID})
+			req = authRequest("PUT", "/api/v1/cart/address", body, user.ID)
+			w = httptest.NewRecorder()
+			handler.SetAddress(w, req)
+			if w.Code != http.StatusOK {
+				t.Fatalf("SetAddress: %d %s", w.Code, w.Body.String())
+			}
+
+			// set card
+			body, _ = json.Marshal(map[string]interface{}{"card_id": card.ID})
+			req = authRequest("PUT", "/api/v1/cart/card", body, user.ID)
+			w = httptest.NewRecorder()
+			handler.SetCard(w, req)
+			if w.Code != http.StatusOK {
+				t.Fatalf("SetCard: %d %s", w.Code, w.Body.String())
+			}
+
+			// Convert
+			req = authRequest("POST", "/api/v1/cart/convert", nil, user.ID)
+			w = httptest.NewRecorder()
+			handler.ConvertCart(w, req)
+
+			var resp struct {
+				Success bool `json:"success"`
+				Error   struct {
+					Code    string                 `json:"code"`
+					Message string                 `json:"message"`
+					Details map[string]interface{} `json:"details"`
+				} `json:"error"`
+			}
+			_ = json.NewDecoder(w.Body).Decode(&resp)
+
+			isOverCap := w.Code == http.StatusBadRequest && resp.Error.Code == "CART_OVER_LIMIT"
+			if isOverCap != tc.expectOverCap {
+				t.Fatalf("expectOverCap=%v got code=%d errCode=%q msg=%q", tc.expectOverCap, w.Code, resp.Error.Code, resp.Error.Message)
+			}
+			if tc.expectOverCap {
+				if got, _ := resp.Error.Details["limit_cents"].(float64); int(got) != tc.capCents {
+					t.Errorf("limit_cents in details: want %d, got %v", tc.capCents, resp.Error.Details["limit_cents"])
+				}
+				if got, _ := resp.Error.Details["total_cents"].(float64); int(got) != tc.quantity*500 {
+					t.Errorf("total_cents in details: want %d, got %v", tc.quantity*500, resp.Error.Details["total_cents"])
+				}
+			}
+		})
+	}
+}

api/main.go

@@ -71,7 +71,7 @@ func main() {
 	jwtManager := auth.NewJWTManager(cfg.JWTSecret, 30*time.Minute)
 
 	// Setup routes
-	router := routes.SetupRoutes(version, cfg.StripeSecretKey, cfg.StripeWebhookSecret, jwtManager, cfg.AuthFingerprintKey, cfg.ShippoAPIKey, cfg.BringAPIUID, cfg.BringAPIKey, cfg.BringCustomerNumber, cfg.ShippoWebhookSecret, cfg.AppURL, cfg.SlackSigningSecret, cfg.AdminAPIKey, cfg.APIPort)
+	router := routes.SetupRoutes(version, cfg.StripeSecretKey, cfg.StripeWebhookSecret, jwtManager, cfg.AuthFingerprintKey, cfg.ShippoAPIKey, cfg.BringAPIUID, cfg.BringAPIKey, cfg.BringCustomerNumber, cfg.ShippoWebhookSecret, cfg.AppURL, cfg.SlackSigningSecret, cfg.AdminAPIKey, cfg.APIPort, cfg.MaxOrderCents)
 
 	// Create HTTP server
 	server := &http.Server{

api/routes/routes.go

@@ -29,6 +29,7 @@ func SetupRoutes(
 	slackSigningSecret string,
 	adminAPIKey string,
 	apiPort string,
+	maxOrderCents int,
 ) *chi.Mux {
 	r := chi.NewRouter()
 
@@ -46,7 +47,7 @@ func SetupRoutes(
 	healthHandler := handlers.NewHealthHandler(version)
 	productHandler := handlers.NewProductHandler()
 	authHandler := handlers.NewAuthHandler(jwtManager, authFingerprintKey)
-	cartHandler := handlers.NewCartHandler(stripeSecretKey, appURL)
+	cartHandler := handlers.NewCartHandler(stripeSecretKey, appURL, maxOrderCents)
 	cardHandler := handlers.NewCardHandler(stripeSecretKey, appURL)
 	orderHandler := handlers.NewOrderHandler(stripeSecretKey, bringAPIUID, bringAPIKey, bringCustomerNumber)
 	addressHandler := handlers.NewAddressHandler(shippoAPIKey, bringAPIUID, bringAPIKey)

pkg/config/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/joho/godotenv"
@@ -30,6 +31,7 @@ type Config struct {
 	ShippoWebhookSecret string
 	SlackSigningSecret  string
 	AdminAPIKey         string
+	MaxOrderCents       int
 }
 
 // Load reads configuration from environment variables and validates required secrets.
@@ -66,6 +68,7 @@ func Load() (*Config, error) {
 		ShippoWebhookSecret: os.Getenv("SHIPPO_WEBHOOK_SECRET"),
 		SlackSigningSecret:  os.Getenv("SLACK_SIGNING_SECRET"),
 		AdminAPIKey:         os.Getenv("ADMIN_API_KEY"),
+		MaxOrderCents:       loadMaxOrderCents(),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -130,3 +133,20 @@ func getEnvOrDefault(key, defaultValue string) string {
 	}
 	return defaultValue
 }
+
+// loadMaxOrderCents reads MAX_ORDER_CENTS. Default $200. Explicit 0 disables
+// the cap. Garbage or negative values fall back to the default with a warning
+// so an operator typo cannot silently disable a fraud control.
+func loadMaxOrderCents() int {
+	const defaultCap = 20000
+	raw := os.Getenv("MAX_ORDER_CENTS")
+	if raw == "" {
+		return defaultCap
+	}
+	n, err := strconv.Atoi(raw)
+	if err != nil || n < 0 {
+		fmt.Printf("WARNING: MAX_ORDER_CENTS=%q invalid, falling back to default %d\n", raw, defaultCap)
+		return defaultCap
+	}
+	return n
+}

pkg/config/config_test.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestLoadMaxOrderCents(t *testing.T) {
+	cases := []struct {
+		name string
+		env  string // "" means unset
+		want int
+	}{
+		{"unset uses default", "", 20000},
+		{"explicit zero disables", "0", 0},
+		{"valid override", "50000", 50000},
+		{"negative falls back to default", "-1", 20000},
+		{"garbage falls back to default", "two hundred", 20000},
+		{"empty after trim falls back to default", "", 20000},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.env == "" {
+				t.Setenv("MAX_ORDER_CENTS", "")
+			} else {
+				t.Setenv("MAX_ORDER_CENTS", tc.env)
+			}
+			got := loadMaxOrderCents()
+			if got != tc.want {
+				t.Errorf("loadMaxOrderCents() = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}

pkg/tui/confirmation.go

@@ -333,6 +333,8 @@ func (m Model) ReviewUpdate(msg tea.Msg) (Model, tea.Cmd) {
 func friendlyCheckoutError(raw error) string {
 	s := raw.Error()
 	switch {
+	case strings.Contains(s, "CART_OVER_LIMIT"):
+		return "order total is over the per-order limit. remove items and try again."
 	case strings.Contains(s, "AUTHENTICATION_REQUIRED"):
 		return "your bank wants to verify this charge, authentication step coming soon."
 	case strings.Contains(s, "AUTHENTICATION_NOT_SUPPORTED"):

scripts/backups/01-log-tail.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Show last 20 lines of the cron log + its mtime. Cheapest "did it run" check.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "Backup log: mtime + last 20 lines"
+spinner_start "fetching log from $SSH_HOST..."
+ssh "$SSH_HOST" "
+  sudo stat -c 'Last modified: %y' '$BACKUP_LOG'
+  echo; sudo tail -20 '$BACKUP_LOG'
+"
+spinner_stop

scripts/backups/02-snapshots.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# List every restic snapshot with timestamps and tags.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "All restic snapshots"
+spinner_start "querying restic via SFTP..."
+ssh "$SSH_HOST" "
+  export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+  export RESTIC_REPOSITORY='$RESTIC_REPO'
+  sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' snapshots
+"
+spinner_stop

scripts/backups/03-check.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# restic repo integrity check.
+#   ./03-check.sh         metadata only (~5s)
+#   ./03-check.sh --deep  + 10% data-pack re-read (~30s)
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+EXTRA=""
+if [ "${1:-}" = "--deep" ]; then
+  EXTRA="--read-data-subset=10%"
+fi
+
+print_header "restic check ${EXTRA:-(metadata only)}"
+spinner_start "running restic check on $SSH_HOST..."
+ssh "$SSH_HOST" "
+  export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+  export RESTIC_REPOSITORY='$RESTIC_REPO'
+  sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' check $EXTRA
+"
+spinner_stop

scripts/backups/04-restore-verify.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Restore the latest snapshot to /tmp on the remote box, then:
+#   - PRAGMA integrity_check on the restored DB
+#   - row-count diff vs the live DB
+# Cleans up /tmp dir at the end.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "Restore latest snapshot and verify SQLite"
+spinner_start "restoring latest snapshot to /tmp on $SSH_HOST..."
+
+ssh "$SSH_HOST" \
+  "PWFILE='$RESTIC_PASSWORD_FILE' \
+   REPO='$RESTIC_REPO' \
+   KEY='$RESTIC_SSH_KEY' \
+   LIVE='$LIVE_DB' \
+   bash -s" <<'REMOTE'
+set -e
+DIR=/tmp/tshop-restore-test
+sudo rm -rf "$DIR" && sudo mkdir -p "$DIR"
+
+sudo -E RESTIC_PASSWORD_FILE="$PWFILE" RESTIC_REPOSITORY="$REPO" \
+  restic -o sftp.args="-i $KEY" restore latest --target "$DIR"
+
+DB=$(sudo find "$DIR" -name 'terminalshop.db' | head -1)
+if [ -z "$DB" ]; then
+  echo 'restored DB not found' >&2
+  exit 1
+fi
+
+echo
+echo '--- restored file ---'
+sudo stat -c '%y  %s bytes  %n' "$DB"
+
+echo
+echo '--- PRAGMA integrity_check ---'
+sudo sqlite3 "$DB" 'PRAGMA integrity_check;'
+
+echo
+echo '--- PRAGMA quick_check ---'
+sudo sqlite3 "$DB" 'PRAGMA quick_check;'
+
+echo
+echo '--- restored vs live row counts ---'
+printf '%-22s %10s %10s\n' table restored live
+for t in $(sudo sqlite3 "$DB" 'SELECT name FROM sqlite_master WHERE type="table";'); do
+  r=$(sudo sqlite3 "$DB" "SELECT COUNT(*) FROM \"$t\";")
+  l=$(sudo sqlite3 "$LIVE" "SELECT COUNT(*) FROM \"$t\";" 2>/dev/null || echo 'n/a')
+  printf '%-22s %10s %10s\n' "$t" "$r" "$l"
+done
+
+sudo rm -rf "$DIR"
+echo
+echo 'OK'
+REMOTE
+spinner_stop

scripts/backups/05-freshness.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Print latest snapshot age. Exit 0 if fresh, 1 if older than STALE_HOURS.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "Latest snapshot freshness"
+
+spinner_start "querying latest snapshot..."
+LATEST=$(ssh "$SSH_HOST" "
+  export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+  export RESTIC_REPOSITORY='$RESTIC_REPO'
+  sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' snapshots --json
+" | jq -r 'max_by(.time) | .time')
+spinner_stop
+
+if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then
+  echo "ERROR: could not parse latest snapshot time" >&2
+  exit 2
+fi
+
+NOW=$(date -u +%s)
+# GNU date (Linux) first, then BSD date (macOS) as fallback.
+if THEN=$(date -u -d "$LATEST" +%s 2>/dev/null); then
+  :
+else
+  CLEAN=$(echo "$LATEST" | sed 's/\.[0-9]*Z$/Z/')
+  THEN=$(date -u -j -f "%Y-%m-%dT%H:%M:%SZ" "$CLEAN" +%s)
+fi
+
+AGE_HRS=$(( (NOW - THEN) / 3600 ))
+echo "Latest snapshot: $LATEST"
+echo "Age:             ${AGE_HRS}h"
+echo "Threshold:       ${STALE_HOURS}h"
+
+if [ "$AGE_HRS" -lt "$STALE_HOURS" ]; then
+  print_ok "snapshot within freshness window"
+else
+  print_fail "STALE — investigate"
+  exit 1
+fi

scripts/backups/06-lfs-direct.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Independent filesystem check on lfs-remote. No restic, no password needed.
+# Proves the repo dir exists, files are present, recent uploads landed,
+# and the backup partition has room.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "lfs-remote filesystem-level repo check"
+spinner_start "inspecting $LFS_HOST filesystem..."
+
+ssh "$LFS_HOST" "
+set -e
+REPO='$LFS_REPO_PATH'
+WARN_PCT=$LFS_DISK_WARN_PCT
+
+if [ ! -d \"\$REPO\" ]; then
+  echo 'ERROR: repo dir missing on lfs' >&2
+  exit 1
+fi
+
+echo '--- snapshot pointer count (one file per restic snapshot) ---'
+COUNT=\$(find \"\$REPO/snapshots\" -type f 2>/dev/null | wc -l)
+echo \"snapshots/: \$COUNT files\"
+
+echo
+echo '--- repo size on disk ---'
+du -sh \"\$REPO\"
+
+echo
+echo '--- recent uploads (files newer than 48h) ---'
+RECENT=\$(find \"\$REPO\" -type f -mtime -2 2>/dev/null | wc -l)
+echo \"files added in last 48h: \$RECENT\"
+if [ \"\$RECENT\" -eq 0 ]; then
+  echo 'WARN: no new files in last 48h' >&2
+fi
+
+echo
+echo '--- most recent snapshot pointer ---'
+LATEST=\$(ls -t \"\$REPO/snapshots\"/* 2>/dev/null | head -1)
+if [ -n \"\$LATEST\" ]; then
+  stat -c '%y  %n' \"\$LATEST\"
+else
+  echo 'WARN: no snapshot files found' >&2
+fi
+
+echo
+echo '--- disk usage ---'
+df -h \"\$REPO\"
+USED=\$(df --output=pcent \"\$REPO\" 2>/dev/null | tail -1 | tr -dc 0-9)
+if [ -n \"\$USED\" ] && [ \"\$USED\" -gt \"\$WARN_PCT\" ]; then
+  echo \"FAIL: backup partition \${USED}% full (threshold \${WARN_PCT}%)\" >&2
+  exit 1
+fi
+
+echo
+echo 'OK'
+"
+spinner_stop

scripts/backups/07-lfs-restic-check.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Independent restic check from the lfs-remote side. Reads repo via a
+# LOCAL filesystem path (not SFTP), so it proves the data is parseable
+# without any involvement from the prod box at restic level.
+#
+# Needs:
+#   - restic installed on lfs-remote (see scratchpkg port at /usr/ports/main/restic)
+#   - SSH access to both prod (to fetch the repo password) and lfs
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "lfs-remote restic check (independent vantage point)"
+
+# Precondition: restic must be on lfs.
+if ! ssh "$LFS_HOST" 'command -v restic >/dev/null 2>&1'; then
+  echo 'ERROR: restic not in PATH on lfs-remote' >&2
+  echo "Install via scratchpkg, then re-run." >&2
+  exit 2
+fi
+
+# Pull restic password from prod (needs sudo on prod).
+spinner_start "fetching restic password from $SSH_HOST..."
+PW=$(ssh "$SSH_HOST" "sudo cat '$RESTIC_PASSWORD_FILE'")
+spinner_stop
+if [ -z "$PW" ]; then
+  echo 'ERROR: could not read restic password from prod' >&2
+  exit 2
+fi
+
+# Run restic against the LOCAL repo path on lfs. Password fed via stdin
+# so it never lands in argv or a file on lfs.
+spinner_start "running restic check from $LFS_HOST..."
+ssh "$LFS_HOST" "restic -r '$LFS_REPO_PATH' --password-file=/dev/stdin check" <<<"$PW"
+spinner_stop
+
+echo
+print_ok "lfs-side restic check passed"

scripts/backups/check-all.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# Master runner. Executes every backup check in sequence.
+#   ./check-all.sh              stop on first failure; interactive menu when TTY
+#   ./check-all.sh --keep-going keep running after failures (no menu)
+#   ./check-all.sh --deep       use --deep on 03-check.sh
+#
+# When stdin and stdout are TTYs, a failed step opens a drill-down menu:
+#   [r]etry  [l]og  [u]nlock  [s]kip  [q]uit
+# Non-TTY (cron / piped) skips the menu and behaves like the old runner.
+set -uo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+KEEP_GOING=0
+DEEP=0
+for arg in "$@"; do
+  case "$arg" in
+    --keep-going) KEEP_GOING=1 ;;
+    --deep)       DEEP=1 ;;
+    -h|--help)
+      echo "Usage: $0 [--keep-going] [--deep]"
+      echo "  --keep-going  don't stop on first failure, suppress interactive menu"
+      echo "  --deep        pass --deep to 03-check.sh (10% data re-read)"
+      exit 0
+      ;;
+    *)
+      echo "unknown flag: $arg" >&2
+      exit 2
+      ;;
+  esac
+done
+
+# Pull in config + color helpers (also gives us SSH_HOST + restic env for unlock).
+source "$SCRIPT_DIR/lib.sh"
+
+declare -a STEP_NAMES=()
+declare -a STEP_STATUS=()   # OK | FAIL
+FAIL=0
+
+prompt_after_fail() {
+  # returns 0=skip, 1=quit, 2=retry
+  local script="$1"
+  while :; do
+    printf '\n%sNext?%s [r]etry  [l]og  [u]nlock  [s]kip  [q]uit > ' "$C_BOLD" "$C_RESET"
+    local choice
+    read -r choice
+    case "$choice" in
+      r|R) return 2 ;;
+      l|L) "$SCRIPT_DIR/01-log-tail.sh"; echo ;;
+      u|U)
+        echo
+        ssh "$SSH_HOST" "
+          export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+          export RESTIC_REPOSITORY='$RESTIC_REPO'
+          sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' unlock
+        "
+        echo
+        ;;
+      s|S) return 0 ;;
+      q|Q) return 1 ;;
+      "")  ;;
+      *)   printf 'unknown: %s\n' "$choice" ;;
+    esac
+  done
+}
+
+run_step() {
+  local script="$1"; shift
+  echo
+  printf '%s########## %s %s ##########%s\n' "$C_CYAN$C_BOLD" "$script" "$*" "$C_RESET"
+
+  local attempts=0
+  while :; do
+    attempts=$((attempts + 1))
+    if "$SCRIPT_DIR/$script" "$@"; then
+      STEP_NAMES+=("$script")
+      STEP_STATUS+=("OK")
+      return 0
+    fi
+
+    printf '%sFAILED:%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$script" >&2
+
+    if [ -t 0 ] && [ -t 1 ] && [ "$KEEP_GOING" -eq 0 ]; then
+      prompt_after_fail "$script"
+      case "$?" in
+        0) STEP_NAMES+=("$script"); STEP_STATUS+=("FAIL"); FAIL=$((FAIL + 1)); return 0 ;;
+        1) STEP_NAMES+=("$script"); STEP_STATUS+=("FAIL"); FAIL=$((FAIL + 1)); print_summary; exit 1 ;;
+        2) echo
+           printf '%s>>>>>> retry %s (attempt %d) <<<<<<%s\n' "$C_CYAN$C_BOLD" "$script" $((attempts + 1)) "$C_RESET"
+           continue ;;
+      esac
+    fi
+
+    # non-interactive path
+    STEP_NAMES+=("$script")
+    STEP_STATUS+=("FAIL")
+    FAIL=$((FAIL + 1))
+    if [ "$KEEP_GOING" -eq 0 ]; then
+      print_summary
+      exit 1
+    fi
+    return 0
+  done
+}
+
+print_summary() {
+  echo
+  printf '%s===============================================%s\n' "$C_CYAN$C_BOLD" "$C_RESET"
+  printf '   %-28s %s\n' "Script" "Result"
+  printf '   %-28s %s\n' "----------------------------" "--------"
+  local i
+  for i in "${!STEP_NAMES[@]}"; do
+    local name="${STEP_NAMES[$i]}"
+    local status="${STEP_STATUS[$i]}"
+    local color sym
+    case "$status" in
+      OK)   color="$C_GREEN"; sym="✓ OK"   ;;
+      FAIL) color="$C_RED"  ; sym="✗ FAIL" ;;
+    esac
+    printf '   %-28s %s%s%s\n' "$name" "$color" "$sym" "$C_RESET"
+  done
+  printf '%s===============================================%s\n' "$C_CYAN$C_BOLD" "$C_RESET"
+  if [ "$FAIL" -eq 0 ]; then
+    printf '%sAll checks passed.%s\n' "$C_GREEN$C_BOLD" "$C_RESET"
+  else
+    printf '%s%d check(s) failed.%s\n' "$C_RED$C_BOLD" "$FAIL" "$C_RESET"
+  fi
+}
+
+run_step 01-log-tail.sh
+run_step 02-snapshots.sh
+if [ "$DEEP" -eq 1 ]; then run_step 03-check.sh --deep; else run_step 03-check.sh; fi
+run_step 04-restore-verify.sh
+run_step 05-freshness.sh
+run_step 06-lfs-direct.sh
+run_step 07-lfs-restic-check.sh
+
+print_summary
+[ "$FAIL" -eq 0 ] || exit 1

scripts/backups/config.env.example

@@ -0,0 +1,28 @@
+# Backup verification config.
+# Copy to config.env (gitignored) and fill in your real values.
+#
+#   cp config.env.example config.env
+#   $EDITOR config.env
+
+# SSH alias for the prod box (resolved via ~/.ssh/config).
+SSH_HOST="ubuntu-helsinki"
+
+# Restic repository URL. Format: sftp:HOST:PATH
+# HOST is the Tailscale name of the backup target box.
+RESTIC_REPO="sftp:YOUR_TAILNET_HOST.ts.net:/root/backups/tshop"
+
+# Paths on the REMOTE prod box (not your laptop).
+RESTIC_PASSWORD_FILE="/root/.restic-password"
+RESTIC_SSH_KEY="/root/.ssh/tshop-backup"
+BACKUP_LOG="/var/log/backup-tshop.log"
+LIVE_DB="/root/terminalShop/data/terminalshop.db"
+
+# Freshness alert threshold (hours). 24h cron + 4h slop = 28.
+STALE_HOURS="28"
+
+# lfs-remote = the box that RECEIVES the backups.
+# Used by 06-lfs-direct.sh and 07-lfs-restic-check.sh for out-of-band checks.
+LFS_HOST="lfs-remote"
+LFS_REPO_PATH="/root/backups/tshop"
+# 06 fails if the backup partition is more full than this percent.
+LFS_DISK_WARN_PCT="80"

scripts/backups/lib.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Shared loader for the backup verification scripts.
+# Each script should: source "$SCRIPT_DIR/lib.sh"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ ! -f "$SCRIPT_DIR/config.env" ]; then
+  echo "ERROR: $SCRIPT_DIR/config.env missing." >&2
+  echo "Run: cp $SCRIPT_DIR/config.env.example $SCRIPT_DIR/config.env" >&2
+  exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$SCRIPT_DIR/config.env"
+
+# ANSI colors. Emit only when stdout is a TTY and NO_COLOR is unset.
+if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
+  C_RED=$'\033[1;31m'
+  C_GREEN=$'\033[1;32m'
+  C_YELLOW=$'\033[1;33m'
+  C_BLUE=$'\033[1;34m'
+  C_CYAN=$'\033[1;36m'
+  C_BOLD=$'\033[1m'
+  C_RESET=$'\033[0m'
+else
+  C_RED=""; C_GREEN=""; C_YELLOW=""; C_BLUE=""; C_CYAN=""; C_BOLD=""; C_RESET=""
+fi
+
+print_header() { printf '\n%s=== %s ===%s\n' "$C_CYAN" "$1" "$C_RESET"; }
+print_ok()     { printf '%sOK%s%s\n'    "$C_GREEN"  "$C_RESET" "${1:+ — $1}"; }
+print_fail()   { printf '%sFAIL%s%s\n'  "$C_RED"    "$C_RESET" "${1:+ — $1}" >&2; }
+print_warn()   { printf '%sWARN%s%s\n'  "$C_YELLOW" "$C_RESET" "${1:+ — $1}" >&2; }
+print_info()   { printf '%s%s%s\n'      "$C_BLUE"   "$1"       "$C_RESET"; }
+
+# Spinner: shows a rotating braille char while a foreground command runs.
+# Use as a pair around the silent/blocking call:
+#   spinner_start "fetching log..."
+#   ssh ...
+#   spinner_stop
+# Auto-skips when stdout isn't a TTY (cron / piped). Real output from the
+# wrapped command will visually overwrite the spinner — by design.
+SPINNER_PID=""
+spinner_start() {
+  [ -t 1 ] && [ -z "${NO_COLOR:-}" ] || return 0
+  local msg="${1:-working}"
+  (
+    trap 'exit 0' TERM
+    local chars=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏')
+    local i=0
+    while :; do
+      i=$(( (i + 1) % ${#chars[@]} ))
+      printf '\r%s%s%s %s' "$C_CYAN" "${chars[$i]}" "$C_RESET" "$msg"
+      sleep 0.08
+    done
+  ) &
+  SPINNER_PID=$!
+  disown "$SPINNER_PID" 2>/dev/null || true
+}
+spinner_stop() {
+  [ -n "$SPINNER_PID" ] || return 0
+  kill "$SPINNER_PID" 2>/dev/null || true
+  wait "$SPINNER_PID" 2>/dev/null || true
+  printf '\r\033[K'
+  SPINNER_PID=""
+}