MrBlake/MrbWebLibre
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

bundle bangs, fetch on demand

Browse Source
This commit is contained in:
Fabian Freund
2025-07-16 16:28:31 +02:00
parent 6a246b5252
commit 009015ab85
12 changed files with 111251 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/assets/bangs/assistant_bangs.json
+122
View File
@@ -0,0 +1,122 @@
[
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "ai",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "as",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "assistant",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "research",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "answer",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "discuss",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "expert",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "llm",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant - Chat",
"d": "kagi.com",
"t": "chat",
"u": "/assistant?q={{{s}}}&internet_access=off",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant - Code",
"d": "kagi.com",
"t": "code",
"u": "/assistant?q={{{s}}}&profile=code",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "custom",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
},
{
"s": "Kagi Assistant",
"d": "kagi.com",
"t": "asst",
"u": "/assistant?q={{{s}}}",
"fmt": [
"url_encode_placeholder",
"url_encode_space_to_plus"
]
}
]
app/assets/bangs/bangs.json
+107351
View File
File diff suppressed because it is too large Load Diff
app/assets/bangs/custom.json
+32
View File
@@ -0,0 +1,32 @@
[
{
"s": "Lensai",
"d": "search.lensai.eu",
"t": "lai",
"u": "https://search.lensai.eu/?q={{{s}}}",
"c": "Online Services",
"sc": "Search"
},
{
"s": "Lensai Search",
"d": "research.lensai.eu",
"t": "lais",
"u": "https://research.lensai.eu/#/s/search/{{{s}}}",
"fmt": [
"url_encode_placeholder"
],
"c": "Online Services",
"sc": "Search"
},
{
"s": "Lensai Answer",
"d": "research.lensai.eu",
"t": "laia",
"u": "https://research.lensai.eu/#/s/answer/{{{s}}}",
"fmt": [
"url_encode_placeholder"
],
"c": "Online Services",
"sc": "Search"
}
]
app/assets/bangs/kagi_bangs.json
+3609
View File
File diff suppressed because it is too large Load Diff
app/assets/bangs/last_sync.txt
+1
View File
@@ -0,0 +1 @@
2025-07-16T13:21:10+00:00
app/lib/domain/services/app_initialization.dart
+1 -1
View File
@@ -37,7 +37,7 @@ class AppInitializationService extends _$AppInitializationService {
return ref
.read(bangSyncRepositoryProvider.notifier)
.syncBangGroups(syncInterval: const Duration(days: 7));
.syncBundledBangGroups();
}
Future<void> initialize() async {
app/lib/features/bangs/data/models/bang_group.dart
+17 -6
View File
@@ -1,16 +1,27 @@
enum BangGroup {
general('https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json'),
general(
remote:
'https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json',
bundled: 'assets/bangs/bangs.json',
),
assistant(
'https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json',
remote:
'https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json',
bundled: 'assets/bangs/assistant_bangs.json',
),
kagi(
'https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json',
remote:
'https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json',
bundled: 'assets/bangs/kagi_bangs.json',
),
custom(
'https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json',
remote:
'https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json',
bundled: 'assets/bangs/custom.json',
);
final String url;
final String bundled;
final String remote;
const BangGroup(this.url);
const BangGroup({required this.bundled, required this.remote});
}
app/lib/features/bangs/data/services/data_source.dart
+24 -1
View File
@@ -2,6 +2,7 @@ import 'dart:convert';
import 'package:exceptions/exceptions.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/services.dart' show rootBundle;
import 'package:http/http.dart' as http;
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:weblibre/core/http_error_handler.dart';
@@ -15,7 +16,7 @@ class BangDataSourceService extends _$BangDataSourceService {
@override
void build() {}
Future<Result<List<Bang>>> getBangs(Uri url, BangGroup? group) {
Future<Result<List<Bang>>> fetchRemoteBangs(Uri url, BangGroup? group) {
return Result.fromAsync(() async {
return await compute((args) async {
final client = http.Client();
@@ -42,4 +43,26 @@ class BangDataSourceService extends _$BangDataSourceService {
);
}, exceptionHandler: handleHttpError);
}
Future<DateTime> getBundledBangDate(String path) async {
final content = await rootBundle.loadString(path);
return DateTime.parse(content.trim()).toLocal();
}
Future<Result<List<Bang>>> getBundledBangs(String path, BangGroup? group) {
return Result.fromAsync(() async {
final content = await rootBundle.loadString(path);
final json = jsonDecode(content) as List;
return json.map((e) {
var bang = Bang.fromJson(e as Map<String, dynamic>);
if (group != null) {
bang = bang.copyWith.group(group);
}
return bang;
}).toList();
});
}
}
app/lib/features/bangs/domain/repositories/sync.dart
+60 -9
View File
@@ -11,7 +11,7 @@ part 'sync.g.dart';
@Riverpod(keepAlive: true)
class BangSyncRepository extends _$BangSyncRepository {
static Future<Result<void>> _fetchAndSync({
static Future<Result<void>> _fetchAndSyncRemote({
required BangDataSourceService sourceService,
required BangDatabase db,
required Uri url,
@@ -29,7 +29,7 @@ class BangSyncRepository extends _$BangSyncRepository {
}
}
final result = await sourceService.getBangs(url, group);
final result = await sourceService.fetchRemoteBangs(url, group);
return result.flatMapAsync((remoteBangs) async {
await db.syncDao.syncBangs(
group: group,
@@ -40,7 +40,37 @@ class BangSyncRepository extends _$BangSyncRepository {
});
}
Future<Result<void>> syncBangGroup(
static Future<Result<void>> _fetchAndSyncBundled({
required BangDataSourceService sourceService,
required BangDatabase db,
required BangGroup group,
}) async {
final lastSync = await db.syncDao
.getLastSyncOfGroup(group)
.getSingleOrNull();
final sourceDate = await sourceService.getBundledBangDate(
'assets/bangs/last_sync.txt',
);
if (lastSync != null &&
(sourceDate == lastSync ||
sourceDate.difference(lastSync).isNegative)) {
return Result.success(null);
}
final result = await sourceService.getBundledBangs(group.bundled, group);
return result.flatMapAsync((remoteBangs) async {
await db.syncDao.syncBangs(
group: group,
remoteBangs: remoteBangs,
syncTime: sourceDate,
);
await db.optimizeFtsIndex();
});
}
Future<Result<void>> syncRemoteBangGroup(
BangGroup group,
Duration? syncInterval,
) async {
@@ -52,12 +82,12 @@ class BangSyncRepository extends _$BangSyncRepository {
connect: BangDatabase.new,
computation: (db) async {
final ref = ProviderContainer();
final result = await _fetchAndSync(
final result = await _fetchAndSyncRemote(
sourceService: ref.read(
bangDataSourceServiceProvider.notifier,
),
db: db,
url: Uri.parse(group.url),
url: Uri.parse(group.remote),
group: group,
syncInterval: syncInterval,
);
@@ -78,6 +108,29 @@ class BangSyncRepository extends _$BangSyncRepository {
}
}
Future<Result<void>> syncBundledBangGroup(BangGroup group) async {
try {
final db = ref.read(bangDatabaseProvider);
final result = await _fetchAndSyncBundled(
sourceService: ref.read(bangDataSourceServiceProvider.notifier),
db: db,
group: group,
);
//Throw if necessary
return result;
} catch (e) {
return Result.failure(
ErrorMessage(
message: "Failed to sync Bangs (${group.name})",
source: 'BangSync',
details: e,
),
);
}
}
Stream<DateTime?> watchLastSyncOfGroup(BangGroup group) {
return ref
.read(bangDatabaseProvider)
@@ -86,18 +139,16 @@ class BangSyncRepository extends _$BangSyncRepository {
.watchSingleOrNull();
}
Future<Map<BangGroup, Result<void>>> syncBangGroups({
Future<Map<BangGroup, Result<void>>> syncBundledBangGroups({
Set<BangGroup>? groups,
Duration? syncInterval,
}) async {
//Default to all sources
groups ??= BangGroup.values.toSet();
//Run isolated operations
final futures = groups.map(
(source) => syncBangGroup(
(source) => syncBundledBangGroup(
source,
syncInterval,
).then((result) => MapEntry(source, result)),
);
app/lib/features/settings/presentation/widgets/bang_group_list_tile.dart
+1 -1
View File
@@ -40,7 +40,7 @@ class BangGroupListTile extends HookConsumerWidget {
onPressed: () async {
await ref
.read(bangSyncRepositoryProvider.notifier)
.syncBangGroup(group, null);
.syncRemoteBangGroup(group, null);
},
icon: const Icon(Icons.sync),
label: const Text('Sync'),
app/pubspec.yaml
+1
View File
@@ -101,6 +101,7 @@ dev_dependencies:
flutter:
uses-material-design: true
assets:
- assets/bangs/
- assets/fonts/
- assets/icon/
- assets/images/
app/scripts/download_bangs.sh
Executable
+32
View File
@@ -0,0 +1,32 @@
#!/bin/bash
# Change to parent directory
cd "$(dirname "$0")/.."
# Create directory structure if it doesn't exist
mkdir -p assets/bangs
# Array of remote URLs and their corresponding local paths
declare -A downloads=(
["https://raw.githubusercontent.com/FaFre/bangs/main/data/bangs.json"]="assets/bangs/bangs.json"
["https://raw.githubusercontent.com/FaFre/bangs/main/data/assistant_bangs.json"]="assets/bangs/assistant_bangs.json"
["https://raw.githubusercontent.com/FaFre/bangs/main/data/kagi_bangs.json"]="assets/bangs/kagi_bangs.json"
["https://raw.githubusercontent.com/FaFre/bangs/refs/heads/custom/data/custom.json"]="assets/bangs/custom.json"
)
# Download each file
for remote_url in "${!downloads[@]}"; do
local_path="${downloads[$remote_url]}"
echo "Downloading $remote_url to $local_path"
if curl -fsSL "$remote_url" -o "$local_path"; then
echo "✓ Successfully downloaded $local_path"
else
echo "✗ Failed to download $local_path"
exit 1
fi
done
# Write current timestamp to sync file
echo "$(date -u --iso-8601=seconds)" > assets/bangs/last_sync.txt
echo "✓ Sync completed at $(cat assets/bangs/last_sync.txt)"