reorganize build scripts
This commit is contained in:
Fabian Freund
@@ -1,32 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Change to parent directory
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Create directory structure if it doesn't exist
|
||||
mkdir -p app/assets/bangs
|
||||
|
||||
# Array of remote URLs and their corresponding local paths
|
||||
declare -A downloads=(
|
||||
["https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json"]="app/assets/bangs/bangs.json"
|
||||
["https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json"]="app/assets/bangs/assistant_bangs.json"
|
||||
["https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json"]="app/assets/bangs/kagi_bangs.json"
|
||||
["https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json"]="app/assets/bangs/custom.json"
|
||||
)
|
||||
|
||||
# Download each file
|
||||
for remote_url in "${!downloads[@]}"; do
|
||||
local_path="${downloads[$remote_url]}"
|
||||
echo "Downloading $remote_url to $local_path"
|
||||
|
||||
if curl -fsSL "$remote_url" -o "$local_path"; then
|
||||
echo "✓ Successfully downloaded $local_path"
|
||||
else
|
||||
echo "✗ Failed to download $local_path"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Write current timestamp to sync file
|
||||
echo "$(date -u --iso-8601=seconds)" > app/assets/bangs/last_sync.txt
|
||||
echo "✓ Sync completed at $(cat app/assets/bangs/last_sync.txt)"
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Unified asset updater — downloads all external data files used by the app.
|
||||
# Usage: ./scripts/update-assets.sh [--group bangs|bridges|url-cleaner|url-shorteners] ...
|
||||
# Without --group flags, all groups are updated.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
MAX_RETRIES=3
|
||||
RETRY_DELAY=5
|
||||
CURL_TIMEOUT=30
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
log() { printf '[INFO] %s\n' "$*"; }
|
||||
err() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
|
||||
# Fetch a URL to a local path with retries and atomic write.
|
||||
# Uses a temp file so a failed download never clobbers an existing asset.
|
||||
fetch() {
|
||||
local url="$1" dest="$2"
|
||||
local tmp="${dest}.tmp.$$"
|
||||
|
||||
mkdir -p "$(dirname "$dest")"
|
||||
|
||||
for attempt in $(seq 1 "$MAX_RETRIES"); do
|
||||
log "Attempt $attempt/$MAX_RETRIES: $url"
|
||||
if curl --proto '=https' --tlsv1.2 -fsSL \
|
||||
--max-time "$CURL_TIMEOUT" \
|
||||
--retry 0 \
|
||||
-o "$tmp" "$url"; then
|
||||
|
||||
# Basic sanity: reject empty files
|
||||
if [ ! -s "$tmp" ]; then
|
||||
err "Downloaded file is empty: $url"
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
fi
|
||||
|
||||
mv "$tmp" "$dest"
|
||||
log "OK: $dest"
|
||||
return 0
|
||||
fi
|
||||
|
||||
err "Failed (attempt $attempt/$MAX_RETRIES): $url"
|
||||
rm -f "$tmp"
|
||||
[ "$attempt" -lt "$MAX_RETRIES" ] && sleep "$RETRY_DELAY"
|
||||
done
|
||||
|
||||
err "Giving up after $MAX_RETRIES attempts: $url"
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── asset groups ─────────────────────────────────────────────────────────────
|
||||
|
||||
update_bangs() {
|
||||
local dir="$REPO_ROOT/app/assets/bangs"
|
||||
log "Updating bangs..."
|
||||
|
||||
fetch "https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json" \
|
||||
"$dir/bangs.json"
|
||||
fetch "https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json" \
|
||||
"$dir/assistant_bangs.json"
|
||||
fetch "https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json" \
|
||||
"$dir/kagi_bangs.json"
|
||||
fetch "https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json" \
|
||||
"$dir/custom.json"
|
||||
|
||||
date -u --iso-8601=seconds > "$dir/last_sync.txt"
|
||||
log "Bangs sync completed at $(cat "$dir/last_sync.txt")"
|
||||
}
|
||||
|
||||
update_bridges() {
|
||||
local dir="$REPO_ROOT/app/assets/preferences"
|
||||
log "Updating Tor bridges..."
|
||||
|
||||
fetch "https://bridges.torproject.org/moat/circumvention/builtin" \
|
||||
"$dir/builtin-bridges.json"
|
||||
}
|
||||
|
||||
update_url_cleaner() {
|
||||
local dir="$REPO_ROOT/app/assets/preferences"
|
||||
log "Updating URL cleaner catalog..."
|
||||
|
||||
fetch "https://rules2.clearurls.xyz/data.minify.json" \
|
||||
"$dir/url_cleaner_data.minify.json"
|
||||
}
|
||||
|
||||
update_url_shorteners() {
|
||||
local dir="$REPO_ROOT/app/assets/preferences"
|
||||
log "Updating URL shortener list..."
|
||||
|
||||
fetch "https://raw.githubusercontent.com/MISP/misp-warninglists/refs/heads/main/lists/url-shortener/list.json" \
|
||||
"$dir/url-shortener-list.json"
|
||||
}
|
||||
|
||||
# ── main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
ALL_GROUPS=(bangs bridges url-cleaner url-shorteners)
|
||||
SELECTED_GROUPS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--group) SELECTED_GROUPS+=("$2"); shift 2 ;;
|
||||
*) err "Unknown option: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ${#SELECTED_GROUPS[@]} -eq 0 ]; then
|
||||
SELECTED_GROUPS=("${ALL_GROUPS[@]}")
|
||||
fi
|
||||
|
||||
FAILURES=0
|
||||
|
||||
for group in "${SELECTED_GROUPS[@]}"; do
|
||||
case "$group" in
|
||||
bangs) update_bangs || ((FAILURES++)) ;;
|
||||
bridges) update_bridges || ((FAILURES++)) ;;
|
||||
url-cleaner) update_url_cleaner || ((FAILURES++)) ;;
|
||||
url-shorteners) update_url_shorteners || ((FAILURES++)) ;;
|
||||
*) err "Unknown group: $group"; ((FAILURES++)) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$FAILURES" -gt 0 ]; then
|
||||
err "$FAILURES group(s) failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "All assets updated successfully."
|
||||
@@ -1,9 +0,0 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
BASE=$(dirname "$0")
|
||||
|
||||
cd "$BASE/../app/assets/preferences" || exit
|
||||
|
||||
rm -f builtin-bridges.json
|
||||
|
||||
wget -O builtin-bridges.json https://bridges.torproject.org/moat/circumvention/builtin
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
set -eu
|
||||
|
||||
BASE=$(dirname "$0")
|
||||
|
||||
cd "$BASE/../app/assets/preferences" || exit
|
||||
|
||||
TARGET="url_cleaner_data.minify.json"
|
||||
TMP_FILE="${TARGET}.tmp"
|
||||
|
||||
if wget -O "$TMP_FILE" "https://rules2.clearurls.xyz/data.minify.json"; then
|
||||
mv "$TMP_FILE" "$TARGET"
|
||||
echo "Updated $TARGET"
|
||||
else
|
||||
rm -f "$TMP_FILE"
|
||||
echo "Failed to fetch $TARGET, keeping existing file" >&2
|
||||
fi
|
||||
@@ -1,18 +0,0 @@
|
||||
#!/usr/bin/env sh
|
||||
|
||||
set -eu
|
||||
|
||||
BASE=$(dirname "$0")
|
||||
|
||||
cd "$BASE/../app/assets/preferences" || exit
|
||||
|
||||
TARGET="url-shortener-list.json"
|
||||
TMP_FILE="${TARGET}.tmp"
|
||||
|
||||
if wget -O "$TMP_FILE" https://raw.githubusercontent.com/MISP/misp-warninglists/refs/heads/main/lists/url-shortener/list.json; then
|
||||
mv "$TMP_FILE" "$TARGET"
|
||||
echo "Updated $TARGET"
|
||||
else
|
||||
rm -f "$TMP_FILE"
|
||||
echo "Failed to fetch $TARGET, keeping existing file" >&2
|
||||
fi
|
||||
Reference in New Issue
Block a user