chore(ops): add nightly backup verification scripts

* feat(sca): handle 3ds checkout end to end with qr poll and race fix

* fix(sca): drop unused pollOrderStatusCmd helper

.github/workflows/ci.yml

@@ -8,9 +8,9 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
 
@@ -30,6 +30,7 @@ jobs:
           fi
 
       - name: Lint
+        if: ${{ !env.ACT }}
         uses: golangci/golangci-lint-action@v7
         with:
           version: v2.4.0

.github/workflows/deploy.yml

@@ -8,10 +8,10 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.23'
+          go-version-file: go.mod
       - name: Run tests
         run: go test ./...
       - name: Deploy to server

.gitignore

@@ -114,3 +114,6 @@ ROADMAP.md
 
 *.bak
 terminalshop.prod.*
+
+# Backup verification config (real values; example file is committed)
+scripts/backups/config.env

api/handlers/cards.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	"sync"
+	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/stripe/stripe-go/v78"
@@ -24,35 +24,43 @@ import (
 	stripeSession "github.com/stripe/stripe-go/v78/checkout/session"
 )
 
-// payRedirects maps short tokens to full Stripe checkout URLs.
+const RedirectTTL = 10 * time.Minute
-var (
-	payRedirects   = map[string]string{}
-	payRedirectsMu sync.Mutex
-)
 
-func storeRedirect(stripeURL string) (string, error) {
+func storeRedirect(stripeURL, purpose string) (string, error) {
 	b := make([]byte, 16)
 	if _, err := rand.Read(b); err != nil {
 		return "", err
 	}
 	token := hex.EncodeToString(b)
-	payRedirectsMu.Lock()
+	now := time.Now()
-	payRedirects[token] = stripeURL
+	rec := models.PayRedirect{
-	payRedirectsMu.Unlock()
+		Token:     token,
+		URL:       stripeURL,
+		Purpose:   purpose,
+		CreatedAt: now,
+		ExpiresAt: now.Add(RedirectTTL),
+	}
+	if err := database.GetDB().Create(&rec).Error; err != nil {
+		return "", err
+	}
 	return token, nil
 }
 
 // PayRedirect resolves a short token and redirects to the Stripe checkout URL.
 func PayRedirect(w http.ResponseWriter, r *http.Request) {
 	token := chi.URLParam(r, "token")
-	payRedirectsMu.Lock()
+	db := database.GetDB().WithContext(r.Context())
-	url, ok := payRedirects[token]
+	var rec models.PayRedirect
-	payRedirectsMu.Unlock()
+	if err := db.Where("token = ?", token).First(&rec).Error; err != nil {
-	if !ok {
 		http.NotFound(w, r)
 		return
 	}
-	http.Redirect(w, r, url, http.StatusFound)
+	if time.Now().After(rec.ExpiresAt) {
+		db.Where("token = ?", rec.Token).Delete(&models.PayRedirect{})
+		http.NotFound(w, r)
+		return
+	}
+	http.Redirect(w, r, rec.URL, http.StatusFound)
 }
 
 // CardHandler handles saved payment method CRUD.
@@ -168,7 +176,7 @@ func (h *CardHandler) CollectCard(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	token, err := storeRedirect(sess.URL)
+	token, err := storeRedirect(sess.URL, models.RedirectPurposeAddCard)
 	if err != nil {
 		utils.RespondError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to generate redirect token", nil)
 		return

api/handlers/cart.go

@@ -23,13 +23,14 @@ import (
 )
 
 type CartHandler struct {
-	stripeKey string
+	stripeKey     string
-	appURL    string
+	appURL        string
+	maxOrderCents int
 }
 
 // NewCartHandler creates a new cart handler with the Stripe secret key
-func NewCartHandler(stripeSecretKey, appURL string) *CartHandler {
+func NewCartHandler(stripeSecretKey, appURL string, maxOrderCents int) *CartHandler {
-	return &CartHandler{stripeKey: stripeSecretKey, appURL: appURL}
+	return &CartHandler{stripeKey: stripeSecretKey, appURL: appURL, maxOrderCents: maxOrderCents}
 }
 
 // getOrCreateCart finds the users cart or creates one.
@@ -354,6 +355,17 @@ func (h *CartHandler) ConvertCart(w http.ResponseWriter, r *http.Request) {
 
 	total := subtotal + cart.ShippingCost
 
+	if h.maxOrderCents > 0 && total > h.maxOrderCents {
+		middleware.RecordCartConversion("validation_over_limit")
+		utils.RespondError(w, http.StatusBadRequest, "CART_OVER_LIMIT",
+			fmt.Sprintf("order total must be at most $%.2f", float64(h.maxOrderCents)/100),
+			map[string]interface{}{
+				"total_cents": total,
+				"limit_cents": h.maxOrderCents,
+			})
+		return
+	}
+
 	if total < 50 {
 		middleware.RecordCartConversion("validation_min_amount")
 		utils.RespondError(w, http.StatusBadRequest, "VALIDATION_ERROR", "order total must be at least $0.50", nil)
@@ -425,7 +437,7 @@ func (h *CartHandler) ConvertCart(w http.ResponseWriter, r *http.Request) {
 		middleware.ObserveStripeRequest("payment_intent_create", "error", stripeDur)
 		if stripeErr, ok := err.(*stripe.Error); ok {
 			if stripeErr.Code == stripe.ErrorCodeAuthenticationRequired && stripeErr.PaymentIntent != nil {
-				h.respondRequiresAction(w, &order, stripeErr.PaymentIntent)
+				h.respondRequiresAction(w, &order, paymentMethodID, stripeErr.PaymentIntent)
 				return
 			}
 			db.Model(&order).Update("status", models.OrderStatusFailed)
@@ -454,7 +466,7 @@ func (h *CartHandler) ConvertCart(w http.ResponseWriter, r *http.Request) {
 	middleware.ObserveStripeRequest("payment_intent_create", "success", stripeDur)
 
 	if pi.Status == stripe.PaymentIntentStatusRequiresAction {
-		h.respondRequiresAction(w, &order, pi)
+		h.respondRequiresAction(w, &order, paymentMethodID, pi)
 		return
 	}
 
@@ -503,9 +515,24 @@ func (h *CartHandler) ConvertCart(w http.ResponseWriter, r *http.Request) {
 // respondRequiresAction handles a PaymentIntent that needs 3DS auth
 // The order stays pending. the webhook flips it to paid once the customer
 // completes the bank-hosted challenge, the tui polls /api/v1/orders/{id}/status
-func (h *CartHandler) respondRequiresAction(w http.ResponseWriter, order *models.Order, pi *stripe.PaymentIntent) {
+func (h *CartHandler) respondRequiresAction(w http.ResponseWriter, order *models.Order, paymentMethodID string, pi *stripe.PaymentIntent) {
 	db := database.GetDB()
 
+	// After an off-session authentication_required decline, the PI sits in
+	// status=requires_payment_method with payment_method detached. Re-confirm
+	// on-session with the same PM so Stripe pushes the PI into requires_action
+	// and populates next_action.redirect_to_url with the 3DS challenge URL.
+	if pi.NextAction == nil || pi.NextAction.RedirectToURL == nil || pi.NextAction.RedirectToURL.URL == "" {
+		confirmed, cerr := paymentintent.Confirm(pi.ID, &stripe.PaymentIntentConfirmParams{
+			PaymentMethod: stripe.String(paymentMethodID),
+			OffSession:    stripe.Bool(false),
+			ReturnURL:     stripe.String(h.appURL + "/post-3ds"),
+		})
+		if cerr == nil && confirmed != nil {
+			pi = confirmed
+		}
+	}
+
 	if pi.NextAction == nil || pi.NextAction.RedirectToURL == nil || pi.NextAction.RedirectToURL.URL == "" {
 		middleware.RecordCartConversion("sca_missing_next_action")
 		db.Model(order).Update("status", models.OrderStatusFailed)
@@ -514,15 +541,19 @@ func (h *CartHandler) respondRequiresAction(w http.ResponseWriter, order *models
 		return
 	}
 
-	// persist the paymentIntent id before responding so the reconciliation jon
+	// Persist the paymentIntent id AND reset status to pending. The
-	// and webhook can match the eventual succeeded/failed event to this order
+	// payment_intent.payment_failed webhook from the initial off-session decline
-	// even if the customer drops off mid-auth
+	// often arrives before this response and flips the order to failed —
-	if err := db.Model(order).Update("stripe_payment_id", pi.ID).Error; err != nil {
+	// override it back so the 3DS-in-flight order shows the correct state.
+	if err := db.Model(order).Updates(map[string]any{
+		"stripe_payment_id": pi.ID,
+		"status":            models.OrderStatusPending,
+	}).Error; err != nil {
 		utils.RespondError(w, http.StatusInternalServerError, "DATABASE_ERROR", "failed to save payment id", nil)
 		return
 	}
 
-	token, err := storeRedirect(pi.NextAction.RedirectToURL.URL)
+	token, err := storeRedirect(pi.NextAction.RedirectToURL.URL, models.RedirectPurposeCheckout3DS)
 	if err != nil {
 		utils.RespondError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to allocate redirect token", nil)
 		return

api/handlers/cart_test.go

@@ -65,7 +65,7 @@ func TestGetCartEmpty(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	req := authRequest("GET", "/api/v1/cart", nil, user.ID)
 	w := httptest.NewRecorder()
@@ -104,7 +104,7 @@ func TestSetItemAndGetCart(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item (CoffeeID=1, Quantity=2)
 	body, _ := json.Marshal(map[string]interface{}{
@@ -160,7 +160,7 @@ func TestSetItemUpdateQuantity(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 1, "quantity": 2})
@@ -205,7 +205,7 @@ func TestSetItemRemove(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 1, "quantity": 3})
@@ -244,7 +244,7 @@ func TestSetItemInvalidProduct(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 999, "quantity": 1})
 	req := authRequest("PUT", "/api/v1/cart/item", body, user.ID)
@@ -261,7 +261,7 @@ func TestSetItemMissingCoffeeID(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"quantity": 1})
 	req := authRequest("PUT", "/api/v1/cart/item", body, user.ID)
@@ -278,7 +278,7 @@ func TestClearCart(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add two items
 	for _, coffeeID := range []uint{1, 2} {
@@ -334,7 +334,7 @@ func TestSetAddress(t *testing.T) {
 	}
 	db.Create(&addr)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"address_id": addr.ID})
 	req := authRequest("PUT", "/api/v1/cart/address", body, user.ID)
@@ -388,7 +388,7 @@ func TestSetAddressWrongUser(t *testing.T) {
 	}
 	db.Create(&addr)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"address_id": addr.ID})
 	req := authRequest("PUT", "/api/v1/cart/address", body, user.ID)
@@ -416,7 +416,7 @@ func TestSetCard(t *testing.T) {
 	}
 	db.Create(&card)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"card_id": card.ID})
 	req := authRequest("PUT", "/api/v1/cart/card", body, user.ID)
@@ -468,7 +468,7 @@ func TestSetCardWrongUser(t *testing.T) {
 	}
 	db.Create(&card)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	body, _ := json.Marshal(map[string]interface{}{"card_id": card.ID})
 	req := authRequest("PUT", "/api/v1/cart/card", body, user.ID)
@@ -485,7 +485,7 @@ func TestConvertCartMissingAddress(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add item but no address or card
 	body, _ := json.Marshal(map[string]interface{}{"coffee_id": 1, "quantity": 1})
@@ -508,7 +508,7 @@ func TestConvertCartMissingCard(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 	db := database.GetDB()
 
 	// Add item and address
@@ -539,7 +539,7 @@ func TestConvertCartEmpty(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Empty cart convert
 	req := authRequest("POST", "/api/v1/cart/convert", nil, user.ID)
@@ -556,7 +556,7 @@ func TestMultipleItems(t *testing.T) {
 	defer func() { _ = os.Remove(testDB) }()
 	defer database.ResetForTesting()
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 
 	// Add three different items
 	for i := uint(1); i <= 3; i++ {
@@ -605,7 +605,7 @@ func TestCartIsolation(t *testing.T) {
 	}
 	db.Create(&user2)
 
-	handler := NewCartHandler("", "")
+	handler := NewCartHandler("", "", 0)
 	r := chi.NewRouter()
 	r.Put("/cart/item", handler.SetItem)
 	r.Get("/cart", handler.GetCart)
@@ -664,3 +664,103 @@ func TestCartIsolation(t *testing.T) {
 		t.Errorf("User 2 expected coffee_id 2, got %d", resp.Data.Cart.Items[0].CoffeeID)
 	}
 }
+
+func TestConvertCartLimit(t *testing.T) {
+	cases := []struct {
+		name          string
+		capCents      int
+		quantity      int
+		expectOverCap bool
+	}{
+		{"over cap rejects", 20000, 41, true},   // $205
+		{"at cap accepts", 20000, 40, false},    // $200 boundary
+		{"under cap accepts", 20000, 39, false}, // $195
+		{"zero cap disables", 0, 1000, false},   // explicit off-switch
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			testDB, user := setupCartTestDB(t)
+			defer func() { _ = os.Remove(testDB) }()
+			defer database.ResetForTesting()
+
+			db := database.GetDB()
+			user.StripeCustomerID = "cus_test_overlimit"
+			if err := db.Save(&user).Error; err != nil {
+				t.Fatalf("saved user: %v", err)
+			}
+
+			addr := models.Address{
+				UserID: user.ID, Name: "Test", Street: "1 Main St",
+				City: "PDX", State: "OR", Zip: "97201", Country: "US",
+			}
+			if err := db.Create(&addr).Error; err != nil {
+				t.Fatalf("seed address: %v", err)
+			}
+
+			card := models.Card{
+				UserID: user.ID, StripePaymentID: "pm_test_overlimit",
+				Last4: "4242", Brand: "Visa", ExpMonth: 12, ExpYear: 2030,
+			}
+			if err := db.Create(&card).Error; err != nil {
+				t.Fatalf("seed card: %v", err)
+			}
+
+			handler := NewCartHandler("", "", tc.capCents)
+
+			body, _ := json.Marshal(map[string]interface{}{"coffee_id": 4, "quantity": tc.quantity})
+			req := authRequest("PUT", "/api/v1/cart/item", body, user.ID)
+			w := httptest.NewRecorder()
+			handler.SetItem(w, req)
+			if w.Code != http.StatusOK {
+				t.Fatalf("SetItem: %d %s", w.Code, w.Body.String())
+			}
+
+			// set address
+			body, _ = json.Marshal(map[string]interface{}{"address_id": addr.ID})
+			req = authRequest("PUT", "/api/v1/cart/address", body, user.ID)
+			w = httptest.NewRecorder()
+			handler.SetAddress(w, req)
+			if w.Code != http.StatusOK {
+				t.Fatalf("SetAddress: %d %s", w.Code, w.Body.String())
+			}
+
+			// set card
+			body, _ = json.Marshal(map[string]interface{}{"card_id": card.ID})
+			req = authRequest("PUT", "/api/v1/cart/card", body, user.ID)
+			w = httptest.NewRecorder()
+			handler.SetCard(w, req)
+			if w.Code != http.StatusOK {
+				t.Fatalf("SetCard: %d %s", w.Code, w.Body.String())
+			}
+
+			// Convert
+			req = authRequest("POST", "/api/v1/cart/convert", nil, user.ID)
+			w = httptest.NewRecorder()
+			handler.ConvertCart(w, req)
+
+			var resp struct {
+				Success bool `json:"success"`
+				Error   struct {
+					Code    string                 `json:"code"`
+					Message string                 `json:"message"`
+					Details map[string]interface{} `json:"details"`
+				} `json:"error"`
+			}
+			_ = json.NewDecoder(w.Body).Decode(&resp)
+
+			isOverCap := w.Code == http.StatusBadRequest && resp.Error.Code == "CART_OVER_LIMIT"
+			if isOverCap != tc.expectOverCap {
+				t.Fatalf("expectOverCap=%v got code=%d errCode=%q msg=%q", tc.expectOverCap, w.Code, resp.Error.Code, resp.Error.Message)
+			}
+			if tc.expectOverCap {
+				if got, _ := resp.Error.Details["limit_cents"].(float64); int(got) != tc.capCents {
+					t.Errorf("limit_cents in details: want %d, got %v", tc.capCents, resp.Error.Details["limit_cents"])
+				}
+				if got, _ := resp.Error.Details["total_cents"].(float64); int(got) != tc.quantity*500 {
+					t.Errorf("total_cents in details: want %d, got %v", tc.quantity*500, resp.Error.Details["total_cents"])
+				}
+			}
+		})
+	}
+}

api/handlers/webhook.go

@@ -167,6 +167,16 @@ func (h *WebhookHandler) handlePaymentIntentFailed(ctx context.Context, event st
 		return
 	}
 
+	// authentication_required is recoverable — the cart handler re-confirms the
+	// PI on-session and the customer completes 3DS via redirect. Marking the
+	// order failed here would race with the recovery path and surface a
+	// momentarily-broken state in the TUI.
+	if pi.LastPaymentError != nil && string(pi.LastPaymentError.Code) == "authentication_required" {
+		webhookLog().Info("ignoring authentication_required payment_failed; handled via 3ds redirect",
+			"order_id", orderIDStr, "pi", pi.ID)
+		return
+	}
+
 	db := database.GetDB().WithContext(ctx)
 	if err := db.Model(&models.Order{}).Where("id = ? AND status = ?", orderIDStr, models.OrderStatusPending).
 		Update("status", models.OrderStatusFailed).Error; err != nil {

api/main.go

@@ -71,7 +71,7 @@ func main() {
 	jwtManager := auth.NewJWTManager(cfg.JWTSecret, 30*time.Minute)
 
 	// Setup routes
-	router := routes.SetupRoutes(version, cfg.StripeSecretKey, cfg.StripeWebhookSecret, jwtManager, cfg.AuthFingerprintKey, cfg.ShippoAPIKey, cfg.BringAPIUID, cfg.BringAPIKey, cfg.BringCustomerNumber, cfg.ShippoWebhookSecret, cfg.AppURL, cfg.SlackSigningSecret, cfg.AdminAPIKey, cfg.APIPort)
+	router := routes.SetupRoutes(version, cfg.StripeSecretKey, cfg.StripeWebhookSecret, jwtManager, cfg.AuthFingerprintKey, cfg.ShippoAPIKey, cfg.BringAPIUID, cfg.BringAPIKey, cfg.BringCustomerNumber, cfg.ShippoWebhookSecret, cfg.AppURL, cfg.SlackSigningSecret, cfg.AdminAPIKey, cfg.APIPort, cfg.MaxOrderCents)
 
 	// Create HTTP server
 	server := &http.Server{

api/routes/routes.go

@@ -29,6 +29,7 @@ func SetupRoutes(
 	slackSigningSecret string,
 	adminAPIKey string,
 	apiPort string,
+	maxOrderCents int,
 ) *chi.Mux {
 	r := chi.NewRouter()
 
@@ -46,7 +47,7 @@ func SetupRoutes(
 	healthHandler := handlers.NewHealthHandler(version)
 	productHandler := handlers.NewProductHandler()
 	authHandler := handlers.NewAuthHandler(jwtManager, authFingerprintKey)
-	cartHandler := handlers.NewCartHandler(stripeSecretKey, appURL)
+	cartHandler := handlers.NewCartHandler(stripeSecretKey, appURL, maxOrderCents)
 	cardHandler := handlers.NewCardHandler(stripeSecretKey, appURL)
 	orderHandler := handlers.NewOrderHandler(stripeSecretKey, bringAPIUID, bringAPIKey, bringCustomerNumber)
 	addressHandler := handlers.NewAddressHandler(shippoAPIKey, bringAPIUID, bringAPIKey)

pkg/api/client.go

@@ -280,14 +280,31 @@ type CardResponse struct {
 }
 
 // ConvertCartResponse represents the cart conversion (checkout) API response.
+// 200 carries Order populated; 202 (requires_action / 3DS) carries the
+// redirect fields and OrderID instead.
 type ConvertCartResponse struct {
 	Success bool `json:"success"`
 	Data    struct {
-		Order models.Order `json:"order"`
+		Order           *models.Order `json:"order,omitempty"`
+		OrderID         uint          `json:"order_id,omitempty"`
+		PaymentIntentID string        `json:"payment_intent_id,omitempty"`
+		Status          string        `json:"status,omitempty"`
+		RedirectToken   string        `json:"redirect_token,omitempty"`
+		RedirectURL     string        `json:"redirect_url,omitempty"`
 	} `json:"data"`
 	Error *APIError `json:"error,omitempty"`
 }
 
+// CheckoutOutcome is the unified result of ConvertCart. RequiresAction=true
+// means the customer needs to complete 3DS at RedirectURL; poll
+// GetOrderStatus(OrderID) until the webhook flips the order to paid or failed.
+type CheckoutOutcome struct {
+	Order          *models.Order
+	OrderID        uint
+	RequiresAction bool
+	RedirectURL    string
+}
+
 // GetCart fetches the user's current cart.
 func (c *Client) GetCart() (*CartData, error) {
 	url := fmt.Sprintf("%s/api/v1/cart", c.BaseURL)
@@ -467,7 +484,9 @@ func (c *Client) ClearCart() error {
 }
 
 // ConvertCart converts the cart into an order, charging the saved card.
-func (c *Client) ConvertCart() (*models.Order, error) {
+// Returns a CheckoutOutcome that either carries the completed Order or
+// indicates the customer must complete 3DS authentication via RedirectURL.
+func (c *Client) ConvertCart() (*CheckoutOutcome, error) {
 	url := fmt.Sprintf("%s/api/v1/cart/convert", c.BaseURL)
 
 	req, err := http.NewRequest("POST", url, nil)
@@ -493,7 +512,21 @@ func (c *Client) ConvertCart() (*models.Order, error) {
 		return nil, fmt.Errorf("failed to convert cart")
 	}
 
-	return &convertResp.Data.Order, nil
+	if convertResp.Data.Status == "requires_action" {
+		return &CheckoutOutcome{
+			OrderID:        convertResp.Data.OrderID,
+			RequiresAction: true,
+			RedirectURL:    convertResp.Data.RedirectURL,
+		}, nil
+	}
+
+	if convertResp.Data.Order == nil {
+		return nil, fmt.Errorf("malformed convert response: missing order")
+	}
+	return &CheckoutOutcome{
+		Order:   convertResp.Data.Order,
+		OrderID: convertResp.Data.Order.ID,
+	}, nil
 }
 
 type OrderStatusResponse struct {

pkg/config/config.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/joho/godotenv"
@@ -30,6 +31,7 @@ type Config struct {
 	ShippoWebhookSecret string
 	SlackSigningSecret  string
 	AdminAPIKey         string
+	MaxOrderCents       int
 }
 
 // Load reads configuration from environment variables and validates required secrets.
@@ -66,6 +68,7 @@ func Load() (*Config, error) {
 		ShippoWebhookSecret: os.Getenv("SHIPPO_WEBHOOK_SECRET"),
 		SlackSigningSecret:  os.Getenv("SLACK_SIGNING_SECRET"),
 		AdminAPIKey:         os.Getenv("ADMIN_API_KEY"),
+		MaxOrderCents:       loadMaxOrderCents(),
 	}
 
 	if err := cfg.validate(); err != nil {
@@ -130,3 +133,20 @@ func getEnvOrDefault(key, defaultValue string) string {
 	}
 	return defaultValue
 }
+
+// loadMaxOrderCents reads MAX_ORDER_CENTS. Default $200. Explicit 0 disables
+// the cap. Garbage or negative values fall back to the default with a warning
+// so an operator typo cannot silently disable a fraud control.
+func loadMaxOrderCents() int {
+	const defaultCap = 20000
+	raw := os.Getenv("MAX_ORDER_CENTS")
+	if raw == "" {
+		return defaultCap
+	}
+	n, err := strconv.Atoi(raw)
+	if err != nil || n < 0 {
+		fmt.Printf("WARNING: MAX_ORDER_CENTS=%q invalid, falling back to default %d\n", raw, defaultCap)
+		return defaultCap
+	}
+	return n
+}

pkg/config/config_test.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"testing"
+)
+
+func TestLoadMaxOrderCents(t *testing.T) {
+	cases := []struct {
+		name string
+		env  string // "" means unset
+		want int
+	}{
+		{"unset uses default", "", 20000},
+		{"explicit zero disables", "0", 0},
+		{"valid override", "50000", 50000},
+		{"negative falls back to default", "-1", 20000},
+		{"garbage falls back to default", "two hundred", 20000},
+		{"empty after trim falls back to default", "", 20000},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.env == "" {
+				t.Setenv("MAX_ORDER_CENTS", "")
+			} else {
+				t.Setenv("MAX_ORDER_CENTS", tc.env)
+			}
+			got := loadMaxOrderCents()
+			if got != tc.want {
+				t.Errorf("loadMaxOrderCents() = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}

pkg/database/migrate.go

@@ -3,6 +3,7 @@ package database
 import (
 	"fmt"
 	"log"
+
 	"terminalShop/pkg/models"
 
 	"gorm.io/gorm"
@@ -24,6 +25,7 @@ func Migrate(db *gorm.DB) error {
 		&models.OrderItem{}, // Line items within orders
 		&models.Address{},   // Saved addresses
 		&models.OrderEvent{},
+		&models.PayRedirect{}, // short-token -> stripe url store (10 min TTL)
 	); err != nil {
 		return fmt.Errorf("migration failed: %w", err)
 	}

pkg/models/pay_redirect.go

@@ -0,0 +1,23 @@
+package models
+
+import "time"
+
+// PayRedirect maps a short token to a stripe hosted url with a TTL.
+// survives api restarts so a customer mid-3Ds authentication still
+// lands on the bank-hosted challenge page after a deploy or crash
+type PayRedirect struct {
+	Token     string    `gorm:"primaryKey;size:64" json:"token"`
+	URL       string    `gorm:"type:text;not null" json:"url"`
+	Purpose   string    `gorm:"size:32;not null" json:"purpose"`
+	CreatedAt time.Time `json:"created_at"`
+	ExpiresAt time.Time `gorm:"index" json:"expires_at"`
+}
+
+const (
+	RedirectPurposeCheckout3DS = "checkout_3ds"
+	RedirectPurposeAddCard     = "add_card"
+)
+
+func (PayRedirect) TableName() string {
+	return "pay_redirects"
+}

pkg/tui/confirmation.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"terminalShop/pkg/models"
+	"terminalShop/pkg/tui/qrfefe"
 
 	"charm.land/bubbles/v2/viewport"
 	tea "charm.land/bubbletea/v2"
@@ -169,6 +170,18 @@ func (m Model) ReviewView() string {
 	if m.CheckingOut {
 		return m.theme.TextAccent().Bold(true).Padding(1).Render("  submitting order...")
 	}
+	if m.review.authFailed {
+		content := m.theme.TextError().Bold(true).Padding(1).Render("authentication failed — press esc to try again")
+		return lipgloss.Place(
+			m.widthContainer,
+			lipgloss.Height(content),
+			lipgloss.Center, lipgloss.Center,
+			content,
+		)
+	}
+	if m.review.requiresAction {
+		return m.renderRequiresActionView()
+	}
 	if m.review.success {
 		content := m.theme.TextSuccess().Bold(true).Padding(1).Render("order placed successfully!")
 		return lipgloss.Place(
@@ -187,6 +200,34 @@ func (m Model) ReviewView() string {
 	)
 }
 
+// renderRequiresActionView shows a QR for the 3DS challenge URL while the TUI
+// polls the order status. Mirrors the add-card QR layout in payment_form.go.
+func (m Model) renderRequiresActionView() string {
+	base := m.theme.TextLabel()
+	accent := m.theme.TextHighlight()
+	header := m.theme.TextAccent().Bold(true).Render("your bank wants to verify this charge")
+
+	if m.review.redirectURL == "" {
+		return base.Render("  waiting for authentication link...")
+	}
+
+	qr, qrSize, err := qrfefe.Generate(m.review.redirectURL)
+	if err != nil || qrSize > m.widthContent {
+		body := header + "\n\n" +
+			base.Render("open this url on your phone to authenticate:") + "\n\n" +
+			accent.Render(m.review.redirectURL) + "\n\n" +
+			base.Render("waiting for completion...")
+		return lipgloss.Place(m.widthContainer, lipgloss.Height(body), lipgloss.Center, lipgloss.Center, body)
+	}
+
+	// Same gotcha as payment_form.go: lipgloss layout helpers corrupt qrfefe ANSI.
+	return qr +
+		header + "\n" +
+		base.Render("scan or visit to complete authentication") + "\n" +
+		accent.Render(m.review.redirectURL) + "\n\n" +
+		base.Render("waiting for completion...")
+}
+
 func (m Model) ReviewUpdate(msg tea.Msg) (Model, tea.Cmd) {
 	switch msg := msg.(type) {
 	case CheckoutResultMsg:
@@ -199,6 +240,12 @@ func (m Model) ReviewUpdate(msg tea.Msg) (Model, tea.Cmd) {
 			m.payment.form = nil
 			return m, nil
 		}
+		if msg.RequiresAction {
+			m.review.requiresAction = true
+			m.review.redirectURL = msg.RedirectURL
+			m.review.orderID = msg.OrderID
+			return m, m.schedulePollOrderStatus(msg.OrderID)
+		}
 		m.Cart = make(map[uint]*models.CartItem)
 		m.cart.cursor = 0
 		m.ShippingInfo = nil
@@ -206,10 +253,57 @@ func (m Model) ReviewUpdate(msg tea.Msg) (Model, tea.Cmd) {
 		m.review.success = true
 		return m, nil
 
+	case OrderStatusMsg:
+		if !m.review.requiresAction || msg.OrderID != m.review.orderID {
+			return m, nil
+		}
+		if msg.Err != nil {
+			// Transient — keep polling.
+			return m, m.schedulePollOrderStatus(msg.OrderID)
+		}
+		switch msg.Status {
+		case "paid", "shipped", "delivered":
+			m.Cart = make(map[uint]*models.CartItem)
+			m.cart.cursor = 0
+			m.ShippingInfo = nil
+			m.OrdersLoaded = false
+			m.review.requiresAction = false
+			m.review.redirectURL = ""
+			m.review.success = true
+			return m, nil
+		case "failed", "cancelled":
+			m.review.requiresAction = false
+			m.review.redirectURL = ""
+			m.review.authFailed = true
+			return m, nil
+		default:
+			return m, m.schedulePollOrderStatus(msg.OrderID)
+		}
+
 	case tea.KeyMsg:
 		if m.CheckingOut {
 			return m, nil
 		}
+		if m.review.authFailed {
+			if msg.String() == "esc" || msg.String() == "enter" {
+				m.review.authFailed = false
+				m = m.SwitchPage(paymentPage)
+				m.payment.view = 0
+				m.payment.form = nil
+				return m, nil
+			}
+			return m, nil
+		}
+		if m.review.requiresAction {
+			if msg.String() == "esc" {
+				m.review.requiresAction = false
+				m.review.redirectURL = ""
+				m = m.SwitchPage(paymentPage)
+				m.payment.view = 0
+				return m, nil
+			}
+			return m, nil
+		}
 		if m.review.success {
 			m.review.success = false
 			m.review.cardJustAdded = false
@@ -239,6 +333,8 @@ func (m Model) ReviewUpdate(msg tea.Msg) (Model, tea.Cmd) {
 func friendlyCheckoutError(raw error) string {
 	s := raw.Error()
 	switch {
+	case strings.Contains(s, "CART_OVER_LIMIT"):
+		return "order total is over the per-order limit. remove items and try again."
 	case strings.Contains(s, "AUTHENTICATION_REQUIRED"):
 		return "your bank wants to verify this charge, authentication step coming soon."
 	case strings.Contains(s, "AUTHENTICATION_NOT_SUPPORTED"):

pkg/tui/model.go

@@ -202,6 +202,15 @@ type reviewState struct {
 	cardJustAdded    bool
 	cardWasDuplicate bool
 	success          bool
+
+	// 3DS / SCA flow: when the backend returns 202 requires_action,
+	// requiresAction is true and redirectURL is the Stripe-hosted challenge
+	// page. The TUI shows a QR and polls GetOrderStatus(orderID) until the
+	// order flips to paid or failed.
+	requiresAction bool
+	redirectURL    string
+	orderID        uint
+	authFailed     bool
 }
 
 type confirmState struct {
@@ -418,8 +427,18 @@ func (m Model) Init() tea.Cmd {
 }
 
 type CheckoutResultMsg struct {
+	OrderID        uint
+	Total          int
+	RequiresAction bool
+	RedirectURL    string
+	Err            error
+}
+
+// OrderStatusMsg carries the result of a poll against /orders/{id}/status.
+// Emitted on the 2s tick while waiting for 3DS authentication to complete.
+type OrderStatusMsg struct {
 	OrderID uint
-	Total   int
+	Status  string
 	Err     error
 }
 
@@ -631,15 +650,38 @@ func (m Model) checkoutWithSavedCard() tea.Cmd {
 		}
 
 		// 3. Convert the cart to an order
-		order, err := m.APIClient.ConvertCart()
+		outcome, err := m.APIClient.ConvertCart()
 		if err != nil {
 			return CheckoutResultMsg{Err: err}
 		}
 
-		return CheckoutResultMsg{OrderID: order.ID, Total: order.Total}
+		if outcome.RequiresAction {
+			return CheckoutResultMsg{
+				OrderID:        outcome.OrderID,
+				RequiresAction: true,
+				RedirectURL:    outcome.RedirectURL,
+			}
+		}
+
+		return CheckoutResultMsg{OrderID: outcome.Order.ID, Total: outcome.Order.Total}
 	}
 }
 
+// schedulePollOrderStatus sleeps 2s then polls /orders/{id}/status. Used while
+// waiting for the customer to complete 3DS authentication.
+func (m Model) schedulePollOrderStatus(orderID uint) tea.Cmd {
+	return tea.Tick(2*time.Second, func(time.Time) tea.Msg {
+		if m.APIClient == nil {
+			return OrderStatusMsg{OrderID: orderID, Err: fmt.Errorf("api client unavailable")}
+		}
+		status, err := m.APIClient.GetOrderStatus(orderID)
+		if err != nil {
+			return OrderStatusMsg{OrderID: orderID, Err: err}
+		}
+		return OrderStatusMsg{OrderID: orderID, Status: status}
+	})
+}
+
 func (m Model) syncCartItemCmd(coffeeID uint, quantity int) (Model, tea.Cmd) {
 	if m.APIClient == nil || m.User == nil {
 		return m, nil

scripts/backups/01-log-tail.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Show last 20 lines of the cron log + its mtime. Cheapest "did it run" check.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "Backup log: mtime + last 20 lines"
+spinner_start "fetching log from $SSH_HOST..."
+ssh "$SSH_HOST" "
+  sudo stat -c 'Last modified: %y' '$BACKUP_LOG'
+  echo; sudo tail -20 '$BACKUP_LOG'
+"
+spinner_stop

scripts/backups/02-snapshots.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# List every restic snapshot with timestamps and tags.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "All restic snapshots"
+spinner_start "querying restic via SFTP..."
+ssh "$SSH_HOST" "
+  export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+  export RESTIC_REPOSITORY='$RESTIC_REPO'
+  sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' snapshots
+"
+spinner_stop

scripts/backups/03-check.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# restic repo integrity check.
+#   ./03-check.sh         metadata only (~5s)
+#   ./03-check.sh --deep  + 10% data-pack re-read (~30s)
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+EXTRA=""
+if [ "${1:-}" = "--deep" ]; then
+  EXTRA="--read-data-subset=10%"
+fi
+
+print_header "restic check ${EXTRA:-(metadata only)}"
+spinner_start "running restic check on $SSH_HOST..."
+ssh "$SSH_HOST" "
+  export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+  export RESTIC_REPOSITORY='$RESTIC_REPO'
+  sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' check $EXTRA
+"
+spinner_stop

scripts/backups/04-restore-verify.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Restore the latest snapshot to /tmp on the remote box, then:
+#   - PRAGMA integrity_check on the restored DB
+#   - row-count diff vs the live DB
+# Cleans up /tmp dir at the end.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "Restore latest snapshot and verify SQLite"
+spinner_start "restoring latest snapshot to /tmp on $SSH_HOST..."
+
+ssh "$SSH_HOST" \
+  "PWFILE='$RESTIC_PASSWORD_FILE' \
+   REPO='$RESTIC_REPO' \
+   KEY='$RESTIC_SSH_KEY' \
+   LIVE='$LIVE_DB' \
+   bash -s" <<'REMOTE'
+set -e
+DIR=/tmp/tshop-restore-test
+sudo rm -rf "$DIR" && sudo mkdir -p "$DIR"
+
+sudo -E RESTIC_PASSWORD_FILE="$PWFILE" RESTIC_REPOSITORY="$REPO" \
+  restic -o sftp.args="-i $KEY" restore latest --target "$DIR"
+
+DB=$(sudo find "$DIR" -name 'terminalshop.db' | head -1)
+if [ -z "$DB" ]; then
+  echo 'restored DB not found' >&2
+  exit 1
+fi
+
+echo
+echo '--- restored file ---'
+sudo stat -c '%y  %s bytes  %n' "$DB"
+
+echo
+echo '--- PRAGMA integrity_check ---'
+sudo sqlite3 "$DB" 'PRAGMA integrity_check;'
+
+echo
+echo '--- PRAGMA quick_check ---'
+sudo sqlite3 "$DB" 'PRAGMA quick_check;'
+
+echo
+echo '--- restored vs live row counts ---'
+printf '%-22s %10s %10s\n' table restored live
+for t in $(sudo sqlite3 "$DB" 'SELECT name FROM sqlite_master WHERE type="table";'); do
+  r=$(sudo sqlite3 "$DB" "SELECT COUNT(*) FROM \"$t\";")
+  l=$(sudo sqlite3 "$LIVE" "SELECT COUNT(*) FROM \"$t\";" 2>/dev/null || echo 'n/a')
+  printf '%-22s %10s %10s\n' "$t" "$r" "$l"
+done
+
+sudo rm -rf "$DIR"
+echo
+echo 'OK'
+REMOTE
+spinner_stop

scripts/backups/05-freshness.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Print latest snapshot age. Exit 0 if fresh, 1 if older than STALE_HOURS.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "Latest snapshot freshness"
+
+spinner_start "querying latest snapshot..."
+LATEST=$(ssh "$SSH_HOST" "
+  export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+  export RESTIC_REPOSITORY='$RESTIC_REPO'
+  sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' snapshots --json
+" | jq -r 'max_by(.time) | .time')
+spinner_stop
+
+if [ -z "$LATEST" ] || [ "$LATEST" = "null" ]; then
+  echo "ERROR: could not parse latest snapshot time" >&2
+  exit 2
+fi
+
+NOW=$(date -u +%s)
+# GNU date (Linux) first, then BSD date (macOS) as fallback.
+if THEN=$(date -u -d "$LATEST" +%s 2>/dev/null); then
+  :
+else
+  CLEAN=$(echo "$LATEST" | sed 's/\.[0-9]*Z$/Z/')
+  THEN=$(date -u -j -f "%Y-%m-%dT%H:%M:%SZ" "$CLEAN" +%s)
+fi
+
+AGE_HRS=$(( (NOW - THEN) / 3600 ))
+echo "Latest snapshot: $LATEST"
+echo "Age:             ${AGE_HRS}h"
+echo "Threshold:       ${STALE_HOURS}h"
+
+if [ "$AGE_HRS" -lt "$STALE_HOURS" ]; then
+  print_ok "snapshot within freshness window"
+else
+  print_fail "STALE — investigate"
+  exit 1
+fi

scripts/backups/06-lfs-direct.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Independent filesystem check on lfs-remote. No restic, no password needed.
+# Proves the repo dir exists, files are present, recent uploads landed,
+# and the backup partition has room.
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "lfs-remote filesystem-level repo check"
+spinner_start "inspecting $LFS_HOST filesystem..."
+
+ssh "$LFS_HOST" "
+set -e
+REPO='$LFS_REPO_PATH'
+WARN_PCT=$LFS_DISK_WARN_PCT
+
+if [ ! -d \"\$REPO\" ]; then
+  echo 'ERROR: repo dir missing on lfs' >&2
+  exit 1
+fi
+
+echo '--- snapshot pointer count (one file per restic snapshot) ---'
+COUNT=\$(find \"\$REPO/snapshots\" -type f 2>/dev/null | wc -l)
+echo \"snapshots/: \$COUNT files\"
+
+echo
+echo '--- repo size on disk ---'
+du -sh \"\$REPO\"
+
+echo
+echo '--- recent uploads (files newer than 48h) ---'
+RECENT=\$(find \"\$REPO\" -type f -mtime -2 2>/dev/null | wc -l)
+echo \"files added in last 48h: \$RECENT\"
+if [ \"\$RECENT\" -eq 0 ]; then
+  echo 'WARN: no new files in last 48h' >&2
+fi
+
+echo
+echo '--- most recent snapshot pointer ---'
+LATEST=\$(ls -t \"\$REPO/snapshots\"/* 2>/dev/null | head -1)
+if [ -n \"\$LATEST\" ]; then
+  stat -c '%y  %n' \"\$LATEST\"
+else
+  echo 'WARN: no snapshot files found' >&2
+fi
+
+echo
+echo '--- disk usage ---'
+df -h \"\$REPO\"
+USED=\$(df --output=pcent \"\$REPO\" 2>/dev/null | tail -1 | tr -dc 0-9)
+if [ -n \"\$USED\" ] && [ \"\$USED\" -gt \"\$WARN_PCT\" ]; then
+  echo \"FAIL: backup partition \${USED}% full (threshold \${WARN_PCT}%)\" >&2
+  exit 1
+fi
+
+echo
+echo 'OK'
+"
+spinner_stop

scripts/backups/07-lfs-restic-check.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Independent restic check from the lfs-remote side. Reads repo via a
+# LOCAL filesystem path (not SFTP), so it proves the data is parseable
+# without any involvement from the prod box at restic level.
+#
+# Needs:
+#   - restic installed on lfs-remote (see scratchpkg port at /usr/ports/main/restic)
+#   - SSH access to both prod (to fetch the repo password) and lfs
+set -euo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPT_DIR/lib.sh"
+
+print_header "lfs-remote restic check (independent vantage point)"
+
+# Precondition: restic must be on lfs.
+if ! ssh "$LFS_HOST" 'command -v restic >/dev/null 2>&1'; then
+  echo 'ERROR: restic not in PATH on lfs-remote' >&2
+  echo "Install via scratchpkg, then re-run." >&2
+  exit 2
+fi
+
+# Pull restic password from prod (needs sudo on prod).
+spinner_start "fetching restic password from $SSH_HOST..."
+PW=$(ssh "$SSH_HOST" "sudo cat '$RESTIC_PASSWORD_FILE'")
+spinner_stop
+if [ -z "$PW" ]; then
+  echo 'ERROR: could not read restic password from prod' >&2
+  exit 2
+fi
+
+# Run restic against the LOCAL repo path on lfs. Password fed via stdin
+# so it never lands in argv or a file on lfs.
+spinner_start "running restic check from $LFS_HOST..."
+ssh "$LFS_HOST" "restic -r '$LFS_REPO_PATH' --password-file=/dev/stdin check" <<<"$PW"
+spinner_stop
+
+echo
+print_ok "lfs-side restic check passed"

scripts/backups/check-all.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+# Master runner. Executes every backup check in sequence.
+#   ./check-all.sh              stop on first failure; interactive menu when TTY
+#   ./check-all.sh --keep-going keep running after failures (no menu)
+#   ./check-all.sh --deep       use --deep on 03-check.sh
+#
+# When stdin and stdout are TTYs, a failed step opens a drill-down menu:
+#   [r]etry  [l]og  [u]nlock  [s]kip  [q]uit
+# Non-TTY (cron / piped) skips the menu and behaves like the old runner.
+set -uo pipefail
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+KEEP_GOING=0
+DEEP=0
+for arg in "$@"; do
+  case "$arg" in
+    --keep-going) KEEP_GOING=1 ;;
+    --deep)       DEEP=1 ;;
+    -h|--help)
+      echo "Usage: $0 [--keep-going] [--deep]"
+      echo "  --keep-going  don't stop on first failure, suppress interactive menu"
+      echo "  --deep        pass --deep to 03-check.sh (10% data re-read)"
+      exit 0
+      ;;
+    *)
+      echo "unknown flag: $arg" >&2
+      exit 2
+      ;;
+  esac
+done
+
+# Pull in config + color helpers (also gives us SSH_HOST + restic env for unlock).
+source "$SCRIPT_DIR/lib.sh"
+
+declare -a STEP_NAMES=()
+declare -a STEP_STATUS=()   # OK | FAIL
+FAIL=0
+
+prompt_after_fail() {
+  # returns 0=skip, 1=quit, 2=retry
+  local script="$1"
+  while :; do
+    printf '\n%sNext?%s [r]etry  [l]og  [u]nlock  [s]kip  [q]uit > ' "$C_BOLD" "$C_RESET"
+    local choice
+    read -r choice
+    case "$choice" in
+      r|R) return 2 ;;
+      l|L) "$SCRIPT_DIR/01-log-tail.sh"; echo ;;
+      u|U)
+        echo
+        ssh "$SSH_HOST" "
+          export RESTIC_PASSWORD_FILE='$RESTIC_PASSWORD_FILE'
+          export RESTIC_REPOSITORY='$RESTIC_REPO'
+          sudo -E restic -o sftp.args='-i $RESTIC_SSH_KEY' unlock
+        "
+        echo
+        ;;
+      s|S) return 0 ;;
+      q|Q) return 1 ;;
+      "")  ;;
+      *)   printf 'unknown: %s\n' "$choice" ;;
+    esac
+  done
+}
+
+run_step() {
+  local script="$1"; shift
+  echo
+  printf '%s########## %s %s ##########%s\n' "$C_CYAN$C_BOLD" "$script" "$*" "$C_RESET"
+
+  local attempts=0
+  while :; do
+    attempts=$((attempts + 1))
+    if "$SCRIPT_DIR/$script" "$@"; then
+      STEP_NAMES+=("$script")
+      STEP_STATUS+=("OK")
+      return 0
+    fi
+
+    printf '%sFAILED:%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$script" >&2
+
+    if [ -t 0 ] && [ -t 1 ] && [ "$KEEP_GOING" -eq 0 ]; then
+      prompt_after_fail "$script"
+      case "$?" in
+        0) STEP_NAMES+=("$script"); STEP_STATUS+=("FAIL"); FAIL=$((FAIL + 1)); return 0 ;;
+        1) STEP_NAMES+=("$script"); STEP_STATUS+=("FAIL"); FAIL=$((FAIL + 1)); print_summary; exit 1 ;;
+        2) echo
+           printf '%s>>>>>> retry %s (attempt %d) <<<<<<%s\n' "$C_CYAN$C_BOLD" "$script" $((attempts + 1)) "$C_RESET"
+           continue ;;
+      esac
+    fi
+
+    # non-interactive path
+    STEP_NAMES+=("$script")
+    STEP_STATUS+=("FAIL")
+    FAIL=$((FAIL + 1))
+    if [ "$KEEP_GOING" -eq 0 ]; then
+      print_summary
+      exit 1
+    fi
+    return 0
+  done
+}
+
+print_summary() {
+  echo
+  printf '%s===============================================%s\n' "$C_CYAN$C_BOLD" "$C_RESET"
+  printf '   %-28s %s\n' "Script" "Result"
+  printf '   %-28s %s\n' "----------------------------" "--------"
+  local i
+  for i in "${!STEP_NAMES[@]}"; do
+    local name="${STEP_NAMES[$i]}"
+    local status="${STEP_STATUS[$i]}"
+    local color sym
+    case "$status" in
+      OK)   color="$C_GREEN"; sym="✓ OK"   ;;
+      FAIL) color="$C_RED"  ; sym="✗ FAIL" ;;
+    esac
+    printf '   %-28s %s%s%s\n' "$name" "$color" "$sym" "$C_RESET"
+  done
+  printf '%s===============================================%s\n' "$C_CYAN$C_BOLD" "$C_RESET"
+  if [ "$FAIL" -eq 0 ]; then
+    printf '%sAll checks passed.%s\n' "$C_GREEN$C_BOLD" "$C_RESET"
+  else
+    printf '%s%d check(s) failed.%s\n' "$C_RED$C_BOLD" "$FAIL" "$C_RESET"
+  fi
+}
+
+run_step 01-log-tail.sh
+run_step 02-snapshots.sh
+if [ "$DEEP" -eq 1 ]; then run_step 03-check.sh --deep; else run_step 03-check.sh; fi
+run_step 04-restore-verify.sh
+run_step 05-freshness.sh
+run_step 06-lfs-direct.sh
+run_step 07-lfs-restic-check.sh
+
+print_summary
+[ "$FAIL" -eq 0 ] || exit 1

scripts/backups/config.env.example

@@ -0,0 +1,28 @@
+# Backup verification config.
+# Copy to config.env (gitignored) and fill in your real values.
+#
+#   cp config.env.example config.env
+#   $EDITOR config.env
+
+# SSH alias for the prod box (resolved via ~/.ssh/config).
+SSH_HOST="ubuntu-helsinki"
+
+# Restic repository URL. Format: sftp:HOST:PATH
+# HOST is the Tailscale name of the backup target box.
+RESTIC_REPO="sftp:YOUR_TAILNET_HOST.ts.net:/root/backups/tshop"
+
+# Paths on the REMOTE prod box (not your laptop).
+RESTIC_PASSWORD_FILE="/root/.restic-password"
+RESTIC_SSH_KEY="/root/.ssh/tshop-backup"
+BACKUP_LOG="/var/log/backup-tshop.log"
+LIVE_DB="/root/terminalShop/data/terminalshop.db"
+
+# Freshness alert threshold (hours). 24h cron + 4h slop = 28.
+STALE_HOURS="28"
+
+# lfs-remote = the box that RECEIVES the backups.
+# Used by 06-lfs-direct.sh and 07-lfs-restic-check.sh for out-of-band checks.
+LFS_HOST="lfs-remote"
+LFS_REPO_PATH="/root/backups/tshop"
+# 06 fails if the backup partition is more full than this percent.
+LFS_DISK_WARN_PCT="80"

scripts/backups/lib.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Shared loader for the backup verification scripts.
+# Each script should: source "$SCRIPT_DIR/lib.sh"
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ ! -f "$SCRIPT_DIR/config.env" ]; then
+  echo "ERROR: $SCRIPT_DIR/config.env missing." >&2
+  echo "Run: cp $SCRIPT_DIR/config.env.example $SCRIPT_DIR/config.env" >&2
+  exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$SCRIPT_DIR/config.env"
+
+# ANSI colors. Emit only when stdout is a TTY and NO_COLOR is unset.
+if [ -t 1 ] && [ -z "${NO_COLOR:-}" ]; then
+  C_RED=$'\033[1;31m'
+  C_GREEN=$'\033[1;32m'
+  C_YELLOW=$'\033[1;33m'
+  C_BLUE=$'\033[1;34m'
+  C_CYAN=$'\033[1;36m'
+  C_BOLD=$'\033[1m'
+  C_RESET=$'\033[0m'
+else
+  C_RED=""; C_GREEN=""; C_YELLOW=""; C_BLUE=""; C_CYAN=""; C_BOLD=""; C_RESET=""
+fi
+
+print_header() { printf '\n%s=== %s ===%s\n' "$C_CYAN" "$1" "$C_RESET"; }
+print_ok()     { printf '%sOK%s%s\n'    "$C_GREEN"  "$C_RESET" "${1:+ — $1}"; }
+print_fail()   { printf '%sFAIL%s%s\n'  "$C_RED"    "$C_RESET" "${1:+ — $1}" >&2; }
+print_warn()   { printf '%sWARN%s%s\n'  "$C_YELLOW" "$C_RESET" "${1:+ — $1}" >&2; }
+print_info()   { printf '%s%s%s\n'      "$C_BLUE"   "$1"       "$C_RESET"; }
+
+# Spinner: shows a rotating braille char while a foreground command runs.
+# Use as a pair around the silent/blocking call:
+#   spinner_start "fetching log..."
+#   ssh ...
+#   spinner_stop
+# Auto-skips when stdout isn't a TTY (cron / piped). Real output from the
+# wrapped command will visually overwrite the spinner — by design.
+SPINNER_PID=""
+spinner_start() {
+  [ -t 1 ] && [ -z "${NO_COLOR:-}" ] || return 0
+  local msg="${1:-working}"
+  (
+    trap 'exit 0' TERM
+    local chars=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏')
+    local i=0
+    while :; do
+      i=$(( (i + 1) % ${#chars[@]} ))
+      printf '\r%s%s%s %s' "$C_CYAN" "${chars[$i]}" "$C_RESET" "$msg"
+      sleep 0.08
+    done
+  ) &
+  SPINNER_PID=$!
+  disown "$SPINNER_PID" 2>/dev/null || true
+}
+spinner_stop() {
+  [ -n "$SPINNER_PID" ] || return 0
+  kill "$SPINNER_PID" 2>/dev/null || true
+  wait "$SPINNER_PID" 2>/dev/null || true
+  printf '\r\033[K'
+  SPINNER_PID=""
+}