small web feature initial
This commit is contained in:
Fabian Freund
@@ -0,0 +1,231 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:isolate';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:rss_dart/dart_rss.dart';
|
||||
import 'package:uuid/enums.dart';
|
||||
import 'package:weblibre/core/logger.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/extensions/http_encoding.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/database.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_feed_entry.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_small_web_mode.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
|
||||
const _staleDuration = Duration(hours: 3);
|
||||
|
||||
typedef _KagiFeedFetchRequest = ({
|
||||
RootIsolateToken token,
|
||||
String url,
|
||||
String mode,
|
||||
Map<String, String> categoryRemap,
|
||||
});
|
||||
|
||||
class KagiSourceService {
|
||||
final SmallWebDatabase _db;
|
||||
final Map<String, String> _categoryRemap;
|
||||
|
||||
KagiSourceService(this._db, this._categoryRemap);
|
||||
|
||||
Future<bool> needsRefresh(KagiSmallWebMode mode) async {
|
||||
final latestFetch = await _db.smallWebItemDao
|
||||
.getLatestFetchedAt(SmallWebSourceKind.kagi, mode)
|
||||
.getSingleOrNull();
|
||||
|
||||
if (latestFetch == null) return true;
|
||||
|
||||
return DateTime.now().difference(latestFetch) > _staleDuration;
|
||||
}
|
||||
|
||||
Future<void> fetchAndIngest(KagiSmallWebMode mode) async {
|
||||
final request = (
|
||||
token: ServicesBinding.rootIsolateToken!,
|
||||
url: mode.feedUrl.toString(),
|
||||
mode: mode.name,
|
||||
categoryRemap: Map<String, String>.from(_categoryRemap),
|
||||
);
|
||||
|
||||
final List<KagiFeedEntry> entries;
|
||||
try {
|
||||
entries = await _runKagiFeedFetch(request);
|
||||
} catch (e, st) {
|
||||
logger.e(
|
||||
'Failed to fetch/parse Kagi feed for $mode',
|
||||
error: e,
|
||||
stackTrace: st,
|
||||
);
|
||||
rethrow;
|
||||
}
|
||||
|
||||
final now = DateTime.now();
|
||||
await _db.batch((batch) {
|
||||
for (final entry in entries) {
|
||||
final itemId = uuid.v5(Namespace.url.value, entry.url.toString());
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebItems,
|
||||
SmallWebItemsCompanion.insert(
|
||||
id: itemId,
|
||||
url: entry.url,
|
||||
title: Value(entry.title),
|
||||
domain: entry.url.host,
|
||||
author: Value(entry.author),
|
||||
summary: Value(entry.summary),
|
||||
publishedAt: Value(entry.publishedAt),
|
||||
createdAt: Value(now),
|
||||
updatedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebItemsCompanion(
|
||||
title: entry.title != null
|
||||
? Value(entry.title)
|
||||
: const Value.absent(),
|
||||
author: entry.author != null
|
||||
? Value(entry.author)
|
||||
: const Value.absent(),
|
||||
summary: entry.summary != null
|
||||
? Value(entry.summary)
|
||||
: const Value.absent(),
|
||||
publishedAt: entry.publishedAt != null
|
||||
? Value(entry.publishedAt)
|
||||
: const Value.absent(),
|
||||
updatedAt: Value(now),
|
||||
),
|
||||
target: [_db.smallWebItems.url],
|
||||
),
|
||||
);
|
||||
|
||||
final membershipId = uuid.v5(
|
||||
Namespace.url.value,
|
||||
'${SmallWebSourceKind.kagi.name}:${mode.name}:${entry.url}',
|
||||
);
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebMemberships,
|
||||
SmallWebMembershipsCompanion.insert(
|
||||
id: membershipId,
|
||||
itemId: itemId,
|
||||
sourceKind: SmallWebSourceKind.kagi,
|
||||
mode: Value(mode.name),
|
||||
consoleUrl: const Value(null),
|
||||
categories: Value(entry.categories),
|
||||
fetchedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebMembershipsCompanion(
|
||||
categories: Value(entry.categories),
|
||||
fetchedAt: Value(now),
|
||||
),
|
||||
target: [_db.smallWebMemberships.id],
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Future<List<KagiFeedEntry>> _runKagiFeedFetch(_KagiFeedFetchRequest request) {
|
||||
return Isolate.run(_createKagiFeedFetchTask(request));
|
||||
}
|
||||
|
||||
Future<List<KagiFeedEntry>> Function() _createKagiFeedFetchTask(
|
||||
_KagiFeedFetchRequest request,
|
||||
) {
|
||||
return () => _fetchAndParseFeed(
|
||||
request.token,
|
||||
Uri.parse(request.url),
|
||||
request.mode,
|
||||
request.categoryRemap,
|
||||
);
|
||||
}
|
||||
|
||||
Future<List<KagiFeedEntry>> _fetchAndParseFeed(
|
||||
RootIsolateToken token,
|
||||
Uri url,
|
||||
String mode,
|
||||
Map<String, String> categoryRemap,
|
||||
) async {
|
||||
BackgroundIsolateBinaryMessenger.ensureInitialized(token);
|
||||
|
||||
final client = http.Client();
|
||||
try {
|
||||
final response = await client.get(url).timeout(const Duration(seconds: 30));
|
||||
|
||||
if (response.statusCode != 200) {
|
||||
throw Exception(
|
||||
'Kagi feed request failed with status ${response.statusCode}',
|
||||
);
|
||||
}
|
||||
|
||||
final xmlString = response.bodyUnicodeFallback;
|
||||
final feed = AtomFeed.parse(xmlString);
|
||||
|
||||
return feed.items
|
||||
.map((item) {
|
||||
final link = item.links
|
||||
.where((l) => l.rel == 'alternate' || l.rel == null)
|
||||
.map((l) => l.href)
|
||||
.firstOrNull;
|
||||
|
||||
final href = link ?? item.links.firstOrNull?.href;
|
||||
final parsedUrl = href != null ? Uri.tryParse(href) : null;
|
||||
if (parsedUrl == null) return null;
|
||||
|
||||
if (mode == 'videos' && parsedUrl.path.contains('/shorts/')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const kagiScheme = 'https://kagi.com/smallweb/categories';
|
||||
final categories = item.categories
|
||||
.where(
|
||||
(c) =>
|
||||
c.scheme == kagiScheme &&
|
||||
c.term != null &&
|
||||
c.term!.isNotEmpty,
|
||||
)
|
||||
.map((c) => categoryRemap[c.term!] ?? c.term!)
|
||||
.toList();
|
||||
|
||||
final author = item.authors
|
||||
.where((a) => a.name != null && a.name!.isNotEmpty)
|
||||
.map((a) => a.name!)
|
||||
.firstOrNull;
|
||||
|
||||
return KagiFeedEntry(
|
||||
url: parsedUrl,
|
||||
title: item.title,
|
||||
author: author,
|
||||
summary: item.summary,
|
||||
publishedAt: item.updated != null
|
||||
? DateTime.tryParse(item.updated!)
|
||||
: null,
|
||||
categories: categories,
|
||||
);
|
||||
})
|
||||
.nonNulls
|
||||
.toList();
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:math';
|
||||
|
||||
import 'package:fast_equatable/fast_equatable.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/database.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_small_web_mode.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/wander_console_source.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/kagi_source_service.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/wander_source_service.dart';
|
||||
|
||||
final _random = Random.secure();
|
||||
|
||||
class WanderDiscoverResult with FastEquatable {
|
||||
final SmallWebItem item;
|
||||
final Uri consoleUrl;
|
||||
|
||||
WanderDiscoverResult({required this.item, required this.consoleUrl});
|
||||
|
||||
@override
|
||||
List<Object?> get hashParameters => [item, consoleUrl];
|
||||
}
|
||||
|
||||
class SmallWebDiscoverService {
|
||||
final SmallWebDatabase _db;
|
||||
final KagiSourceService _kagiService;
|
||||
final WanderSourceService _wanderService;
|
||||
|
||||
SmallWebDiscoverService(this._db, this._kagiService, this._wanderService);
|
||||
|
||||
Future<void> recordVisit({
|
||||
required String itemId,
|
||||
required SmallWebSourceKind sourceKind,
|
||||
required KagiSmallWebMode? mode,
|
||||
Uri? consoleUrl,
|
||||
}) async {
|
||||
await _db.smallWebVisitDao.insertVisit(
|
||||
SmallWebVisit(
|
||||
id: uuid.v4(),
|
||||
itemId: itemId,
|
||||
sourceKind: sourceKind,
|
||||
mode: mode?.name,
|
||||
consoleUrl: consoleUrl,
|
||||
visitedAt: DateTime.now(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<SmallWebItem?> discoverKagi({
|
||||
required KagiSmallWebMode mode,
|
||||
String? category,
|
||||
}) async {
|
||||
if (await _kagiService.needsRefresh(mode)) {
|
||||
await _kagiService.fetchAndIngest(mode);
|
||||
}
|
||||
|
||||
final items = await _db.smallWebItemDao
|
||||
.getDiscoverableKagiItems(mode, category)
|
||||
.get();
|
||||
|
||||
if (items.isEmpty) return null;
|
||||
|
||||
final picked = items[_random.nextInt(items.length)];
|
||||
|
||||
await recordVisit(
|
||||
itemId: picked.id,
|
||||
sourceKind: SmallWebSourceKind.kagi,
|
||||
mode: mode,
|
||||
);
|
||||
|
||||
return picked;
|
||||
}
|
||||
|
||||
Future<WanderDiscoverResult?> discoverWander({
|
||||
Uri? currentConsoleUrl,
|
||||
bool forceNewConsole = false,
|
||||
}) async {
|
||||
await _wanderService.syncSeeds();
|
||||
|
||||
final recentItemIds =
|
||||
(await _db.smallWebVisitDao
|
||||
.getRecentItemIds(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
mode: null,
|
||||
)
|
||||
.get())
|
||||
.toSet();
|
||||
|
||||
// Pick a console to explore
|
||||
Uri consoleUrl;
|
||||
if (currentConsoleUrl != null && !forceNewConsole) {
|
||||
consoleUrl = currentConsoleUrl;
|
||||
} else {
|
||||
final consoleUrls = await _wanderService.getDiscoveredConsoleUrls();
|
||||
|
||||
if (forceNewConsole && currentConsoleUrl != null) {
|
||||
consoleUrls.remove(currentConsoleUrl);
|
||||
}
|
||||
|
||||
if (consoleUrls.isEmpty) return null;
|
||||
|
||||
consoleUrl = consoleUrls[_random.nextInt(consoleUrls.length)];
|
||||
}
|
||||
|
||||
final pages = await _refreshAndGetPages(consoleUrl, forceRetry: true);
|
||||
final unvisitedPages = pages
|
||||
.where((page) => !recentItemIds.contains(page.id))
|
||||
.toList();
|
||||
|
||||
if (unvisitedPages.isNotEmpty) {
|
||||
return _pickAndRecord(unvisitedPages, consoleUrl);
|
||||
}
|
||||
|
||||
// No unvisited pages on this console — try alternatives
|
||||
final result = await _tryAlternativeConsoles(consoleUrl, recentItemIds);
|
||||
if (result != null) return result;
|
||||
|
||||
// Last resort: revisit a page from the original console
|
||||
if (pages.isEmpty) return null;
|
||||
return _pickAndRecord(pages, consoleUrl);
|
||||
}
|
||||
|
||||
Future<void> updateItemTitle(String itemId, String title) {
|
||||
return _db.smallWebItemDao.updateTitle(itemId, title);
|
||||
}
|
||||
|
||||
Future<List<SmallWebItem>> _refreshAndGetPages(
|
||||
Uri consoleUrl, {
|
||||
bool forceRetry = false,
|
||||
}) async {
|
||||
if (await _wanderService.shouldRefreshConsole(
|
||||
consoleUrl,
|
||||
forceRetry: forceRetry,
|
||||
)) {
|
||||
await _wanderService.fetchAndIngestConsole(
|
||||
consoleUrl,
|
||||
source: WanderConsoleSource.discovered,
|
||||
);
|
||||
}
|
||||
return _wanderService.getPagesForConsole(consoleUrl);
|
||||
}
|
||||
|
||||
Future<WanderDiscoverResult?> _tryAlternativeConsoles(
|
||||
Uri excludeConsole,
|
||||
Set<String> recentItemIds,
|
||||
) async {
|
||||
final allConsoles = await _wanderService.getDiscoveredConsoleUrls()
|
||||
..remove(excludeConsole)
|
||||
..shuffle(_random);
|
||||
|
||||
for (final altConsole in allConsoles.take(10)) {
|
||||
try {
|
||||
final altPages = await _refreshAndGetPages(altConsole);
|
||||
final unvisited = altPages
|
||||
.where((p) => !recentItemIds.contains(p.id))
|
||||
.toList();
|
||||
final candidates = unvisited.isNotEmpty ? unvisited : altPages;
|
||||
|
||||
if (candidates.isNotEmpty) {
|
||||
return _pickAndRecord(candidates, altConsole);
|
||||
}
|
||||
} catch (_) {
|
||||
// Skip consoles that fail to fetch; continue trying others.
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
Future<WanderDiscoverResult> _pickAndRecord(
|
||||
List<SmallWebItem> candidates,
|
||||
Uri consoleUrl,
|
||||
) async {
|
||||
final picked = candidates[_random.nextInt(candidates.length)];
|
||||
|
||||
await recordVisit(
|
||||
itemId: picked.id,
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
mode: null,
|
||||
consoleUrl: consoleUrl,
|
||||
);
|
||||
|
||||
return WanderDiscoverResult(item: picked, consoleUrl: consoleUrl);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import 'package:weblibre/extensions/uri.dart';
|
||||
|
||||
final _lineCommentPattern = RegExp(r'(?<!:)//.*$', multiLine: true);
|
||||
final _blockCommentPattern = RegExp(r'/\*[\s\S]*?\*/');
|
||||
|
||||
final _consolesPattern = RegExp(r'consoles\s*:\s*\[([\s\S]*?)\]', dotAll: true);
|
||||
final _pagesPattern = RegExp(r'pages\s*:\s*\[([\s\S]*?)\]', dotAll: true);
|
||||
// ignore: unnecessary_raw_strings
|
||||
final _stringPattern = RegExp(r'''(?:["'`])([^"'`]+)(?:["'`])''');
|
||||
|
||||
List<String> _extractArray(String source, RegExp pattern) {
|
||||
final match = pattern.firstMatch(source);
|
||||
if (match == null) return [];
|
||||
|
||||
final arrayContent = match.group(1) ?? '';
|
||||
return _stringPattern
|
||||
.allMatches(arrayContent)
|
||||
.map((m) => m.group(1)!)
|
||||
.toList();
|
||||
}
|
||||
|
||||
Uri _normalizeUrl(String url) {
|
||||
var normalized = url;
|
||||
|
||||
if (normalized.endsWith('/index.html')) {
|
||||
normalized = normalized.substring(
|
||||
0,
|
||||
normalized.length - 'index.html'.length,
|
||||
);
|
||||
}
|
||||
|
||||
return Uri.parse(normalized);
|
||||
}
|
||||
|
||||
class WanderJsResult {
|
||||
final List<Uri> consoles;
|
||||
final List<Uri> pages;
|
||||
|
||||
const WanderJsResult({required this.consoles, required this.pages});
|
||||
|
||||
factory WanderJsResult.parse(String jsSource) {
|
||||
final cleaned = jsSource
|
||||
.replaceAll(_blockCommentPattern, '')
|
||||
.replaceAll(_lineCommentPattern, '');
|
||||
|
||||
final consoles = _extractArray(cleaned, _consolesPattern);
|
||||
final pages = _extractArray(cleaned, _pagesPattern);
|
||||
|
||||
return WanderJsResult(
|
||||
consoles: consoles
|
||||
.map(_normalizeUrl)
|
||||
.where((uri) => uri.isHttpOrHttps)
|
||||
.toList(),
|
||||
pages: pages
|
||||
.map(_normalizeUrl)
|
||||
.where((uri) => uri.isHttpOrHttps)
|
||||
.toList(),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:isolate';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:uuid/enums.dart';
|
||||
import 'package:weblibre/core/logger.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/database.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/wander_console_source.dart';
|
||||
import 'package:weblibre/features/small_web/data/wander_seed_consoles.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/wander_js_parser.dart';
|
||||
|
||||
const _staleDuration = Duration(hours: 3);
|
||||
const _retryAfterError = Duration(minutes: 30);
|
||||
|
||||
typedef _WanderJsFetchRequest = ({RootIsolateToken token, String url});
|
||||
|
||||
class WanderSourceService {
|
||||
final SmallWebDatabase _db;
|
||||
|
||||
WanderSourceService(this._db);
|
||||
|
||||
Future<bool> shouldRefreshConsole(
|
||||
Uri consoleUrl, {
|
||||
bool forceRetry = false,
|
||||
}) async {
|
||||
final console = await _db.wanderConsoleDao
|
||||
.getConsole(consoleUrl)
|
||||
.getSingleOrNull();
|
||||
|
||||
if (console == null || console.lastFetchedAt == null) return true;
|
||||
|
||||
final age = DateTime.now().difference(console.lastFetchedAt!);
|
||||
if (console.lastFetchFailed == true) {
|
||||
return forceRetry || age > _retryAfterError;
|
||||
}
|
||||
|
||||
return age > _staleDuration;
|
||||
}
|
||||
|
||||
Future<void> syncSeeds() async {
|
||||
final now = DateTime.now();
|
||||
await _db.batch((batch) {
|
||||
for (final seedUrl in wanderSeedConsoles) {
|
||||
final url = Uri.parse(seedUrl);
|
||||
batch.insert(
|
||||
_db.wanderConsoles,
|
||||
WanderConsolesCompanion.insert(
|
||||
url: url,
|
||||
wanderJsUrl: url.resolve('wander.js'),
|
||||
source: WanderConsoleSource.seed,
|
||||
createdAt: Value(now),
|
||||
),
|
||||
onConflict: DoNothing(),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> fetchAndIngestConsole(
|
||||
Uri consoleUrl, {
|
||||
required WanderConsoleSource source,
|
||||
}) async {
|
||||
final wanderJsUrl = consoleUrl.resolve('wander.js');
|
||||
final now = DateTime.now();
|
||||
|
||||
try {
|
||||
final result = await _runWanderJsFetch((
|
||||
token: ServicesBinding.rootIsolateToken!,
|
||||
url: wanderJsUrl.toString(),
|
||||
));
|
||||
|
||||
if (result == null) {
|
||||
await _saveConsoleWithError(consoleUrl, wanderJsUrl, now, source);
|
||||
return null;
|
||||
}
|
||||
|
||||
final existingUrls = await _db.wanderConsoleDao
|
||||
.getExistingConsoleUrls(result.consoles)
|
||||
.get();
|
||||
|
||||
await _db.transaction(() async {
|
||||
await _db.wanderConsoleDao.upsertConsole(
|
||||
WanderConsole(
|
||||
url: consoleUrl,
|
||||
wanderJsUrl: wanderJsUrl,
|
||||
lastFetchedAt: now,
|
||||
lastFetchFailed: false,
|
||||
source: source,
|
||||
createdAt: now,
|
||||
),
|
||||
);
|
||||
|
||||
await _db.batch((batch) {
|
||||
for (final neighborUrl in result.consoles) {
|
||||
batch.insert(
|
||||
_db.wanderConsoleNeighbors,
|
||||
WanderConsoleNeighborsCompanion.insert(
|
||||
sourceConsoleUrl: consoleUrl.toString(),
|
||||
targetConsoleUrl: neighborUrl.toString(),
|
||||
discoveredAt: Value(now),
|
||||
),
|
||||
onConflict: DoNothing(),
|
||||
);
|
||||
|
||||
if (!existingUrls.contains(neighborUrl)) {
|
||||
batch.insert(
|
||||
_db.wanderConsoles,
|
||||
WanderConsolesCompanion.insert(
|
||||
url: neighborUrl,
|
||||
wanderJsUrl: neighborUrl.resolve('wander.js'),
|
||||
discoveredFromUrl: Value(consoleUrl),
|
||||
source: WanderConsoleSource.discovered,
|
||||
createdAt: Value(now),
|
||||
),
|
||||
onConflict: DoNothing(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (final pageUrl in result.pages) {
|
||||
final itemId = uuid.v5(Namespace.url.value, pageUrl.toString());
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebItems,
|
||||
SmallWebItemsCompanion.insert(
|
||||
id: itemId,
|
||||
url: pageUrl,
|
||||
domain: pageUrl.host,
|
||||
createdAt: Value(now),
|
||||
updatedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebItemsCompanion(updatedAt: Value(now)),
|
||||
target: [_db.smallWebItems.url],
|
||||
),
|
||||
);
|
||||
|
||||
final membershipId = uuid.v5(
|
||||
Namespace.url.value,
|
||||
'${SmallWebSourceKind.wander.name}:$consoleUrl:$pageUrl',
|
||||
);
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebMemberships,
|
||||
SmallWebMembershipsCompanion.insert(
|
||||
id: membershipId,
|
||||
itemId: itemId,
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
consoleUrl: Value(consoleUrl),
|
||||
fetchedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebMembershipsCompanion(fetchedAt: Value(now)),
|
||||
target: [_db.smallWebMemberships.id],
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (e, st) {
|
||||
logger.e(
|
||||
'Failed to fetch wander.js from $wanderJsUrl',
|
||||
error: e,
|
||||
stackTrace: st,
|
||||
);
|
||||
await _saveConsoleWithError(consoleUrl, wanderJsUrl, now, source);
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalizes a user-input URL to a wander console URL.
|
||||
///
|
||||
/// Accepts URLs like:
|
||||
/// - `https://example.com/wander/` → kept as-is
|
||||
/// - `https://example.com/wander` → trailing slash added
|
||||
/// - `https://example.com` → `/wander/` appended
|
||||
/// - `https://example.com/` → `wander/` appended
|
||||
///
|
||||
/// Returns the normalized console URL (always ends with `/wander/`).
|
||||
static Uri normalizeConsoleUrl(Uri url) {
|
||||
var path = url.path;
|
||||
|
||||
// Strip trailing wander.js if someone pasted the full JS URL
|
||||
if (path.endsWith('/wander.js')) {
|
||||
path = path.substring(0, path.length - 'wander.js'.length);
|
||||
}
|
||||
|
||||
// Ensure the path ends with /wander/
|
||||
if (!path.endsWith('/wander/')) {
|
||||
if (path.endsWith('/wander')) {
|
||||
path = '$path/';
|
||||
} else {
|
||||
if (!path.endsWith('/')) {
|
||||
path = '$path/';
|
||||
}
|
||||
path = '${path}wander/';
|
||||
}
|
||||
}
|
||||
|
||||
return url.replace(path: path);
|
||||
}
|
||||
|
||||
/// Checks if a console URL already exists in the database.
|
||||
Future<bool> consoleExists(Uri consoleUrl) async {
|
||||
final console = await _db.wanderConsoleDao
|
||||
.getConsole(consoleUrl)
|
||||
.getSingleOrNull();
|
||||
|
||||
return console != null;
|
||||
}
|
||||
|
||||
/// Validates that a URL points to a valid wander console by fetching its
|
||||
/// wander.js and checking it contains valid consoles or pages data.
|
||||
///
|
||||
/// Returns the parsed [WanderJsResult] if valid, or throws with a
|
||||
/// descriptive error message.
|
||||
Future<WanderJsResult> validateConsole(Uri consoleUrl) async {
|
||||
final wanderJsUrl = consoleUrl.resolve('wander.js');
|
||||
|
||||
final result = await _runWanderJsFetch((
|
||||
token: ServicesBinding.rootIsolateToken!,
|
||||
url: wanderJsUrl.toString(),
|
||||
));
|
||||
|
||||
if (result == null) {
|
||||
throw Exception('Could not fetch wander.js from $wanderJsUrl');
|
||||
}
|
||||
|
||||
if (result.consoles.isEmpty && result.pages.isEmpty) {
|
||||
throw Exception('The wander.js file contains no consoles or pages');
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Validates and adds a user-provided console URL.
|
||||
///
|
||||
/// The URL is normalized, checked for duplicates, validated by fetching
|
||||
/// wander.js, then ingested into the database.
|
||||
///
|
||||
/// Returns the normalized console URL.
|
||||
Future<Uri> addConsoleFromUrl(Uri rawUrl) async {
|
||||
final consoleUrl = normalizeConsoleUrl(rawUrl);
|
||||
|
||||
if (await consoleExists(consoleUrl)) {
|
||||
throw Exception('This console has already been added');
|
||||
}
|
||||
|
||||
// Validate by fetching wander.js
|
||||
await validateConsole(consoleUrl);
|
||||
|
||||
// Now do the full ingest
|
||||
await fetchAndIngestConsole(consoleUrl, source: WanderConsoleSource.manual);
|
||||
|
||||
return consoleUrl;
|
||||
}
|
||||
|
||||
Future<List<Uri>> getDiscoveredConsoleUrls() {
|
||||
return _db.wanderConsoleDao.getDiscoveredConsoleUrls().get();
|
||||
}
|
||||
|
||||
Future<List<SmallWebItem>> getPagesForConsole(Uri consoleUrl) {
|
||||
return _db.definitionsDrift
|
||||
.getWanderPagesForConsole(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
consoleUrl: consoleUrl.toString(),
|
||||
)
|
||||
.get();
|
||||
}
|
||||
|
||||
Future<void> _saveConsoleWithError(
|
||||
Uri consoleUrl,
|
||||
Uri wanderJsUrl,
|
||||
DateTime now,
|
||||
WanderConsoleSource source,
|
||||
) async {
|
||||
await _db.wanderConsoleDao.upsertConsole(
|
||||
WanderConsole(
|
||||
url: consoleUrl,
|
||||
wanderJsUrl: wanderJsUrl,
|
||||
lastFetchedAt: now,
|
||||
lastFetchFailed: true,
|
||||
source: source,
|
||||
createdAt: now,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> _runWanderJsFetch(_WanderJsFetchRequest request) {
|
||||
return Isolate.run(_createWanderJsFetchTask(request));
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> Function() _createWanderJsFetchTask(
|
||||
_WanderJsFetchRequest request,
|
||||
) {
|
||||
return () => _fetchAndParseWanderJs(request.token, Uri.parse(request.url));
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> _fetchAndParseWanderJs(
|
||||
RootIsolateToken token,
|
||||
Uri url,
|
||||
) async {
|
||||
BackgroundIsolateBinaryMessenger.ensureInitialized(token);
|
||||
|
||||
final client = http.Client();
|
||||
try {
|
||||
final response = await client.get(url).timeout(const Duration(seconds: 15));
|
||||
|
||||
if (response.statusCode != 200) return null;
|
||||
|
||||
return WanderJsResult.parse(response.body);
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user