From df55e9e8aff99fa66bcec3560496e6b9d932a9c6 Mon Sep 17 00:00:00 2001
From: Fabian Freund <github@fabi.online>
Date: Fri, 20 Mar 2026 04:26:01 +0100
Subject: [PATCH] reorganize build scripts

---
 .github/workflows/build.yml           |   8 +-
 pubspec.yaml                          |  17 ++--
 scripts/download_bangs.sh             |  32 -------
 scripts/update-assets.sh              | 131 ++++++++++++++++++++++++++
 scripts/update-bridges.sh             |   9 --
 scripts/update-url-cleaner-catalog.sh |  18 ----
 scripts/update-url-shorteners.sh      |  18 ----
 7 files changed, 147 insertions(+), 86 deletions(-)
 delete mode 100755 scripts/download_bangs.sh
 create mode 100755 scripts/update-assets.sh
 delete mode 100755 scripts/update-bridges.sh
 delete mode 100755 scripts/update-url-cleaner-catalog.sh
 delete mode 100755 scripts/update-url-shorteners.sh

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 88232139..9bf64d03 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -79,7 +79,13 @@ jobs:
           dart pub global activate melos
           melos bootstrap
 
-      - name: Build browser
+      - name: Update external assets
+        run: melos run update-assets --no-select
+
+      - name: Build components
+        run: melos run build-components --no-select
+
+      - name: Build APK
         run: melos run build-browser --no-select
 
       - name: Build App Bundle
diff --git a/pubspec.yaml b/pubspec.yaml
index 510e99cd..113e1a8c 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -67,18 +67,19 @@ melos:
     format:
       description: Format all Dart files in the workspace
       exec: dart format .
-    build-browser:
-      description: Run all preparation scripts and build APK + App Bundle
+    update-assets:
+      description: Download external assets (bangs, bridges, URL lists)
+      run: bash scripts/update-assets.sh
+    build-components:
+      description: Build JS components and generate quotes DB
       run: |
         set -e
-        echo "=== Running preparation scripts ==="
         sh scripts/build-container-proxy.sh
         sh scripts/build-readability.sh
-        bash scripts/download_bangs.sh
         python3 scripts/build_quotes_db.py
-        sh scripts/update-bridges.sh
-        sh scripts/update-url-cleaner-catalog.sh
-        sh scripts/update-url-shorteners.sh
-        echo "=== Building APK ==="
+    build-browser:
+      description: Build release APK split per ABI
+      run: |
+        set -e
         cd app
         flutter build apk --release --target-platform android-arm,android-arm64 --split-per-abi --no-tree-shake-icons
diff --git a/scripts/download_bangs.sh b/scripts/download_bangs.sh
deleted file mode 100755
index c50f78fe..00000000
--- a/scripts/download_bangs.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-# Change to parent directory
-cd "$(dirname "$0")/.."
-
-# Create directory structure if it doesn't exist
-mkdir -p app/assets/bangs
-
-# Array of remote URLs and their corresponding local paths
-declare -A downloads=(
-    ["https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json"]="app/assets/bangs/bangs.json"
-    ["https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json"]="app/assets/bangs/assistant_bangs.json"
-    ["https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json"]="app/assets/bangs/kagi_bangs.json"
-    ["https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json"]="app/assets/bangs/custom.json"
-)
-
-# Download each file
-for remote_url in "${!downloads[@]}"; do
-    local_path="${downloads[$remote_url]}"
-    echo "Downloading $remote_url to $local_path"
-
-    if curl -fsSL "$remote_url" -o "$local_path"; then
-        echo "✓ Successfully downloaded $local_path"
-    else
-        echo "✗ Failed to download $local_path"
-        exit 1
-    fi
-done
-
-# Write current timestamp to sync file
-echo "$(date -u --iso-8601=seconds)" > app/assets/bangs/last_sync.txt
-echo "✓ Sync completed at $(cat app/assets/bangs/last_sync.txt)"
diff --git a/scripts/update-assets.sh b/scripts/update-assets.sh
new file mode 100755
index 00000000..ac5e9330
--- /dev/null
+++ b/scripts/update-assets.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+#
+# Unified asset updater — downloads all external data files used by the app.
+# Usage: ./scripts/update-assets.sh [--group bangs|bridges|url-cleaner|url-shorteners] ...
+#   Without --group flags, all groups are updated.
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+MAX_RETRIES=3
+RETRY_DELAY=5
+CURL_TIMEOUT=30
+
+# ── helpers ──────────────────────────────────────────────────────────────────
+
+log()  { printf '[INFO]  %s\n' "$*"; }
+err()  { printf '[ERROR] %s\n' "$*" >&2; }
+
+# Fetch a URL to a local path with retries and atomic write.
+# Uses a temp file so a failed download never clobbers an existing asset.
+fetch() {
+  local url="$1" dest="$2"
+  local tmp="${dest}.tmp.$$"
+
+  mkdir -p "$(dirname "$dest")"
+
+  for attempt in $(seq 1 "$MAX_RETRIES"); do
+    log "Attempt $attempt/$MAX_RETRIES: $url"
+    if curl --proto '=https' --tlsv1.2 -fsSL \
+         --max-time "$CURL_TIMEOUT" \
+         --retry 0 \
+         -o "$tmp" "$url"; then
+
+      # Basic sanity: reject empty files
+      if [ ! -s "$tmp" ]; then
+        err "Downloaded file is empty: $url"
+        rm -f "$tmp"
+        return 1
+      fi
+
+      mv "$tmp" "$dest"
+      log "OK: $dest"
+      return 0
+    fi
+
+    err "Failed (attempt $attempt/$MAX_RETRIES): $url"
+    rm -f "$tmp"
+    [ "$attempt" -lt "$MAX_RETRIES" ] && sleep "$RETRY_DELAY"
+  done
+
+  err "Giving up after $MAX_RETRIES attempts: $url"
+  return 1
+}
+
+# ── asset groups ─────────────────────────────────────────────────────────────
+
+update_bangs() {
+  local dir="$REPO_ROOT/app/assets/bangs"
+  log "Updating bangs..."
+
+  fetch "https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json" \
+        "$dir/bangs.json"
+  fetch "https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json" \
+        "$dir/assistant_bangs.json"
+  fetch "https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json" \
+        "$dir/kagi_bangs.json"
+  fetch "https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json" \
+        "$dir/custom.json"
+
+  date -u --iso-8601=seconds > "$dir/last_sync.txt"
+  log "Bangs sync completed at $(cat "$dir/last_sync.txt")"
+}
+
+update_bridges() {
+  local dir="$REPO_ROOT/app/assets/preferences"
+  log "Updating Tor bridges..."
+
+  fetch "https://bridges.torproject.org/moat/circumvention/builtin" \
+        "$dir/builtin-bridges.json"
+}
+
+update_url_cleaner() {
+  local dir="$REPO_ROOT/app/assets/preferences"
+  log "Updating URL cleaner catalog..."
+
+  fetch "https://rules2.clearurls.xyz/data.minify.json" \
+        "$dir/url_cleaner_data.minify.json"
+}
+
+update_url_shorteners() {
+  local dir="$REPO_ROOT/app/assets/preferences"
+  log "Updating URL shortener list..."
+
+  fetch "https://raw.githubusercontent.com/MISP/misp-warninglists/refs/heads/main/lists/url-shortener/list.json" \
+        "$dir/url-shortener-list.json"
+}
+
+# ── main ─────────────────────────────────────────────────────────────────────
+
+ALL_GROUPS=(bangs bridges url-cleaner url-shorteners)
+SELECTED_GROUPS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --group) SELECTED_GROUPS+=("$2"); shift 2 ;;
+    *) err "Unknown option: $1"; exit 1 ;;
+  esac
+done
+
+if [ ${#SELECTED_GROUPS[@]} -eq 0 ]; then
+  SELECTED_GROUPS=("${ALL_GROUPS[@]}")
+fi
+
+FAILURES=0
+
+for group in "${SELECTED_GROUPS[@]}"; do
+  case "$group" in
+    bangs)          update_bangs          || ((FAILURES++)) ;;
+    bridges)        update_bridges        || ((FAILURES++)) ;;
+    url-cleaner)    update_url_cleaner    || ((FAILURES++)) ;;
+    url-shorteners) update_url_shorteners || ((FAILURES++)) ;;
+    *) err "Unknown group: $group"; ((FAILURES++)) ;;
+  esac
+done
+
+if [ "$FAILURES" -gt 0 ]; then
+  err "$FAILURES group(s) failed"
+  exit 1
+fi
+
+log "All assets updated successfully."
diff --git a/scripts/update-bridges.sh b/scripts/update-bridges.sh
deleted file mode 100755
index eb7beca6..00000000
--- a/scripts/update-bridges.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env sh
-
-BASE=$(dirname "$0")
-
-cd "$BASE/../app/assets/preferences" || exit
-
-rm -f builtin-bridges.json
-
-wget -O builtin-bridges.json https://bridges.torproject.org/moat/circumvention/builtin
\ No newline at end of file
diff --git a/scripts/update-url-cleaner-catalog.sh b/scripts/update-url-cleaner-catalog.sh
deleted file mode 100755
index 1a9e97ca..00000000
--- a/scripts/update-url-cleaner-catalog.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-BASE=$(dirname "$0")
-
-cd "$BASE/../app/assets/preferences" || exit
-
-TARGET="url_cleaner_data.minify.json"
-TMP_FILE="${TARGET}.tmp"
-
-if wget -O "$TMP_FILE" "https://rules2.clearurls.xyz/data.minify.json"; then
-  mv "$TMP_FILE" "$TARGET"
-  echo "Updated $TARGET"
-else
-  rm -f "$TMP_FILE"
-  echo "Failed to fetch $TARGET, keeping existing file" >&2
-fi
diff --git a/scripts/update-url-shorteners.sh b/scripts/update-url-shorteners.sh
deleted file mode 100755
index a86b4931..00000000
--- a/scripts/update-url-shorteners.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env sh
-
-set -eu
-
-BASE=$(dirname "$0")
-
-cd "$BASE/../app/assets/preferences" || exit
-
-TARGET="url-shortener-list.json"
-TMP_FILE="${TARGET}.tmp"
-
-if wget -O "$TMP_FILE" https://raw.githubusercontent.com/MISP/misp-warninglists/refs/heads/main/lists/url-shortener/list.json; then
-  mv "$TMP_FILE" "$TARGET"
-  echo "Updated $TARGET"
-else
-  rm -f "$TMP_FILE"
-  echo "Failed to fetch $TARGET, keeping existing file" >&2
-fi