small web feature initial
This commit is contained in:
Fabian Freund
@@ -0,0 +1,137 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:rxdart/rxdart.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_small_web_mode.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
import 'package:weblibre/features/small_web/data/providers.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/kagi_source_service.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/small_web_discover_service.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/wander_source_service.dart';
|
||||
|
||||
part 'providers.g.dart';
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
Future<KagiSourceService> kagiSourceService(Ref ref) async {
|
||||
final categories = await ref.watch(kagiCategoriesProvider.future);
|
||||
return KagiSourceService(
|
||||
ref.watch(smallWebDatabaseProvider),
|
||||
categories.remap,
|
||||
);
|
||||
}
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
WanderSourceService wanderSourceService(Ref ref) {
|
||||
return WanderSourceService(ref.watch(smallWebDatabaseProvider));
|
||||
}
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
Future<SmallWebDiscoverService> smallWebDiscoverService(Ref ref) async {
|
||||
final kagiService = await ref.watch(kagiSourceServiceProvider.future);
|
||||
|
||||
return SmallWebDiscoverService(
|
||||
ref.watch(smallWebDatabaseProvider),
|
||||
kagiService,
|
||||
ref.watch(wanderSourceServiceProvider),
|
||||
);
|
||||
}
|
||||
|
||||
@Riverpod()
|
||||
Stream<List<GetRecentVisitsResult>> smallWebRecentVisits(
|
||||
Ref ref,
|
||||
SmallWebSourceKind sourceKind,
|
||||
KagiSmallWebMode? mode,
|
||||
) {
|
||||
final db = ref.watch(smallWebDatabaseProvider);
|
||||
return db.smallWebVisitDao
|
||||
.getRecentVisits(sourceKind: sourceKind, mode: mode)
|
||||
.watch();
|
||||
}
|
||||
|
||||
@Riverpod()
|
||||
Stream<Map<KagiSmallWebMode, int>> smallWebAllModeItemCounts(Ref ref) {
|
||||
final db = ref.watch(smallWebDatabaseProvider);
|
||||
return db.definitionsDrift.getAllModeItemCounts().watch().map((rows) {
|
||||
final counts = <KagiSmallWebMode, int>{};
|
||||
for (final row in rows) {
|
||||
if (row.mode == null) continue;
|
||||
final mode = KagiSmallWebMode.values
|
||||
.where((m) => m.name == row.mode)
|
||||
.firstOrNull;
|
||||
if (mode != null) counts[mode] = (counts[mode] ?? 0) + row.c;
|
||||
}
|
||||
return counts;
|
||||
});
|
||||
}
|
||||
|
||||
@Riverpod()
|
||||
Stream<({int linkedConsoles, int pages})> wanderConsoleStats(
|
||||
Ref ref,
|
||||
Uri consoleUrl,
|
||||
) {
|
||||
final db = ref.watch(smallWebDatabaseProvider);
|
||||
|
||||
final linkedConsoles = db.definitionsDrift
|
||||
.getConsoleNeighborCount(consoleUrl: consoleUrl.toString())
|
||||
.watchSingle();
|
||||
|
||||
final pages = db.definitionsDrift
|
||||
.getWanderPagesForConsole(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
consoleUrl: consoleUrl.toString(),
|
||||
)
|
||||
.watch();
|
||||
|
||||
return CombineLatestStream.combine2(
|
||||
linkedConsoles,
|
||||
pages,
|
||||
(a, b) => (linkedConsoles: a, pages: b.length),
|
||||
);
|
||||
}
|
||||
|
||||
@Riverpod()
|
||||
Stream<List<GetNeighborConsolesWithPageCountsResult>> wanderNeighborConsoles(
|
||||
Ref ref,
|
||||
Uri consoleUrl,
|
||||
) {
|
||||
final db = ref.watch(smallWebDatabaseProvider);
|
||||
|
||||
return db.definitionsDrift
|
||||
.getNeighborConsolesWithPageCounts(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
sourceConsoleUrl: consoleUrl.toString(),
|
||||
)
|
||||
.watch();
|
||||
}
|
||||
|
||||
@Riverpod()
|
||||
Stream<List<GetAllConsolesWithPageCountsResult>> wanderAllConsoles(
|
||||
Ref ref,
|
||||
String query,
|
||||
) {
|
||||
final db = ref.watch(smallWebDatabaseProvider);
|
||||
return db.definitionsDrift
|
||||
.getAllConsolesWithPageCounts(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
query: query,
|
||||
)
|
||||
.watch();
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'providers.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
// ignore_for_file: type=lint, type=warning
|
||||
|
||||
@ProviderFor(kagiSourceService)
|
||||
final kagiSourceServiceProvider = KagiSourceServiceProvider._();
|
||||
|
||||
final class KagiSourceServiceProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<KagiSourceService>,
|
||||
KagiSourceService,
|
||||
FutureOr<KagiSourceService>
|
||||
>
|
||||
with
|
||||
$FutureModifier<KagiSourceService>,
|
||||
$FutureProvider<KagiSourceService> {
|
||||
KagiSourceServiceProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'kagiSourceServiceProvider',
|
||||
isAutoDispose: false,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$kagiSourceServiceHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$FutureProviderElement<KagiSourceService> $createElement(
|
||||
$ProviderPointer pointer,
|
||||
) => $FutureProviderElement(pointer);
|
||||
|
||||
@override
|
||||
FutureOr<KagiSourceService> create(Ref ref) {
|
||||
return kagiSourceService(ref);
|
||||
}
|
||||
}
|
||||
|
||||
String _$kagiSourceServiceHash() => r'7f0cca556d22cc65356660f3e59c976a35d40d37';
|
||||
|
||||
@ProviderFor(wanderSourceService)
|
||||
final wanderSourceServiceProvider = WanderSourceServiceProvider._();
|
||||
|
||||
final class WanderSourceServiceProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
WanderSourceService,
|
||||
WanderSourceService,
|
||||
WanderSourceService
|
||||
>
|
||||
with $Provider<WanderSourceService> {
|
||||
WanderSourceServiceProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'wanderSourceServiceProvider',
|
||||
isAutoDispose: false,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$wanderSourceServiceHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$ProviderElement<WanderSourceService> $createElement(
|
||||
$ProviderPointer pointer,
|
||||
) => $ProviderElement(pointer);
|
||||
|
||||
@override
|
||||
WanderSourceService create(Ref ref) {
|
||||
return wanderSourceService(ref);
|
||||
}
|
||||
|
||||
/// {@macro riverpod.override_with_value}
|
||||
Override overrideWithValue(WanderSourceService value) {
|
||||
return $ProviderOverride(
|
||||
origin: this,
|
||||
providerOverride: $SyncValueProvider<WanderSourceService>(value),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
String _$wanderSourceServiceHash() =>
|
||||
r'80574ea0c51f56325940edb0fa7e849784ba7421';
|
||||
|
||||
@ProviderFor(smallWebDiscoverService)
|
||||
final smallWebDiscoverServiceProvider = SmallWebDiscoverServiceProvider._();
|
||||
|
||||
final class SmallWebDiscoverServiceProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<SmallWebDiscoverService>,
|
||||
SmallWebDiscoverService,
|
||||
FutureOr<SmallWebDiscoverService>
|
||||
>
|
||||
with
|
||||
$FutureModifier<SmallWebDiscoverService>,
|
||||
$FutureProvider<SmallWebDiscoverService> {
|
||||
SmallWebDiscoverServiceProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'smallWebDiscoverServiceProvider',
|
||||
isAutoDispose: false,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$smallWebDiscoverServiceHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$FutureProviderElement<SmallWebDiscoverService> $createElement(
|
||||
$ProviderPointer pointer,
|
||||
) => $FutureProviderElement(pointer);
|
||||
|
||||
@override
|
||||
FutureOr<SmallWebDiscoverService> create(Ref ref) {
|
||||
return smallWebDiscoverService(ref);
|
||||
}
|
||||
}
|
||||
|
||||
String _$smallWebDiscoverServiceHash() =>
|
||||
r'14fba9eb1c2aa51fa1fac2c2c764ea400603e8ca';
|
||||
|
||||
@ProviderFor(smallWebRecentVisits)
|
||||
final smallWebRecentVisitsProvider = SmallWebRecentVisitsFamily._();
|
||||
|
||||
final class SmallWebRecentVisitsProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<List<GetRecentVisitsResult>>,
|
||||
List<GetRecentVisitsResult>,
|
||||
Stream<List<GetRecentVisitsResult>>
|
||||
>
|
||||
with
|
||||
$FutureModifier<List<GetRecentVisitsResult>>,
|
||||
$StreamProvider<List<GetRecentVisitsResult>> {
|
||||
SmallWebRecentVisitsProvider._({
|
||||
required SmallWebRecentVisitsFamily super.from,
|
||||
required (SmallWebSourceKind, KagiSmallWebMode?) super.argument,
|
||||
}) : super(
|
||||
retry: null,
|
||||
name: r'smallWebRecentVisitsProvider',
|
||||
isAutoDispose: true,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$smallWebRecentVisitsHash();
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return r'smallWebRecentVisitsProvider'
|
||||
''
|
||||
'$argument';
|
||||
}
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$StreamProviderElement<List<GetRecentVisitsResult>> $createElement(
|
||||
$ProviderPointer pointer,
|
||||
) => $StreamProviderElement(pointer);
|
||||
|
||||
@override
|
||||
Stream<List<GetRecentVisitsResult>> create(Ref ref) {
|
||||
final argument = this.argument as (SmallWebSourceKind, KagiSmallWebMode?);
|
||||
return smallWebRecentVisits(ref, argument.$1, argument.$2);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is SmallWebRecentVisitsProvider && other.argument == argument;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
return argument.hashCode;
|
||||
}
|
||||
}
|
||||
|
||||
String _$smallWebRecentVisitsHash() =>
|
||||
r'ec7d837523b856a6e7e828c0f26a97913e2af34a';
|
||||
|
||||
final class SmallWebRecentVisitsFamily extends $Family
|
||||
with
|
||||
$FunctionalFamilyOverride<
|
||||
Stream<List<GetRecentVisitsResult>>,
|
||||
(SmallWebSourceKind, KagiSmallWebMode?)
|
||||
> {
|
||||
SmallWebRecentVisitsFamily._()
|
||||
: super(
|
||||
retry: null,
|
||||
name: r'smallWebRecentVisitsProvider',
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
isAutoDispose: true,
|
||||
);
|
||||
|
||||
SmallWebRecentVisitsProvider call(
|
||||
SmallWebSourceKind sourceKind,
|
||||
KagiSmallWebMode? mode,
|
||||
) => SmallWebRecentVisitsProvider._(argument: (sourceKind, mode), from: this);
|
||||
|
||||
@override
|
||||
String toString() => r'smallWebRecentVisitsProvider';
|
||||
}
|
||||
|
||||
@ProviderFor(smallWebAllModeItemCounts)
|
||||
final smallWebAllModeItemCountsProvider = SmallWebAllModeItemCountsProvider._();
|
||||
|
||||
final class SmallWebAllModeItemCountsProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<Map<KagiSmallWebMode, int>>,
|
||||
Map<KagiSmallWebMode, int>,
|
||||
Stream<Map<KagiSmallWebMode, int>>
|
||||
>
|
||||
with
|
||||
$FutureModifier<Map<KagiSmallWebMode, int>>,
|
||||
$StreamProvider<Map<KagiSmallWebMode, int>> {
|
||||
SmallWebAllModeItemCountsProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'smallWebAllModeItemCountsProvider',
|
||||
isAutoDispose: true,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$smallWebAllModeItemCountsHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$StreamProviderElement<Map<KagiSmallWebMode, int>> $createElement(
|
||||
$ProviderPointer pointer,
|
||||
) => $StreamProviderElement(pointer);
|
||||
|
||||
@override
|
||||
Stream<Map<KagiSmallWebMode, int>> create(Ref ref) {
|
||||
return smallWebAllModeItemCounts(ref);
|
||||
}
|
||||
}
|
||||
|
||||
String _$smallWebAllModeItemCountsHash() =>
|
||||
r'64d50f9fe581003d6f1e7df61b859118a5ad31fa';
|
||||
|
||||
@ProviderFor(wanderConsoleStats)
|
||||
final wanderConsoleStatsProvider = WanderConsoleStatsFamily._();
|
||||
|
||||
final class WanderConsoleStatsProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<({int linkedConsoles, int pages})>,
|
||||
({int linkedConsoles, int pages}),
|
||||
Stream<({int linkedConsoles, int pages})>
|
||||
>
|
||||
with
|
||||
$FutureModifier<({int linkedConsoles, int pages})>,
|
||||
$StreamProvider<({int linkedConsoles, int pages})> {
|
||||
WanderConsoleStatsProvider._({
|
||||
required WanderConsoleStatsFamily super.from,
|
||||
required Uri super.argument,
|
||||
}) : super(
|
||||
retry: null,
|
||||
name: r'wanderConsoleStatsProvider',
|
||||
isAutoDispose: true,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$wanderConsoleStatsHash();
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return r'wanderConsoleStatsProvider'
|
||||
''
|
||||
'($argument)';
|
||||
}
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$StreamProviderElement<({int linkedConsoles, int pages})> $createElement(
|
||||
$ProviderPointer pointer,
|
||||
) => $StreamProviderElement(pointer);
|
||||
|
||||
@override
|
||||
Stream<({int linkedConsoles, int pages})> create(Ref ref) {
|
||||
final argument = this.argument as Uri;
|
||||
return wanderConsoleStats(ref, argument);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is WanderConsoleStatsProvider && other.argument == argument;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
return argument.hashCode;
|
||||
}
|
||||
}
|
||||
|
||||
String _$wanderConsoleStatsHash() =>
|
||||
r'201d2b0669b878762c3d535f39f6d04e030b8112';
|
||||
|
||||
final class WanderConsoleStatsFamily extends $Family
|
||||
with
|
||||
$FunctionalFamilyOverride<
|
||||
Stream<({int linkedConsoles, int pages})>,
|
||||
Uri
|
||||
> {
|
||||
WanderConsoleStatsFamily._()
|
||||
: super(
|
||||
retry: null,
|
||||
name: r'wanderConsoleStatsProvider',
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
isAutoDispose: true,
|
||||
);
|
||||
|
||||
WanderConsoleStatsProvider call(Uri consoleUrl) =>
|
||||
WanderConsoleStatsProvider._(argument: consoleUrl, from: this);
|
||||
|
||||
@override
|
||||
String toString() => r'wanderConsoleStatsProvider';
|
||||
}
|
||||
|
||||
@ProviderFor(wanderNeighborConsoles)
|
||||
final wanderNeighborConsolesProvider = WanderNeighborConsolesFamily._();
|
||||
|
||||
final class WanderNeighborConsolesProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<List<GetNeighborConsolesWithPageCountsResult>>,
|
||||
List<GetNeighborConsolesWithPageCountsResult>,
|
||||
Stream<List<GetNeighborConsolesWithPageCountsResult>>
|
||||
>
|
||||
with
|
||||
$FutureModifier<List<GetNeighborConsolesWithPageCountsResult>>,
|
||||
$StreamProvider<List<GetNeighborConsolesWithPageCountsResult>> {
|
||||
WanderNeighborConsolesProvider._({
|
||||
required WanderNeighborConsolesFamily super.from,
|
||||
required Uri super.argument,
|
||||
}) : super(
|
||||
retry: null,
|
||||
name: r'wanderNeighborConsolesProvider',
|
||||
isAutoDispose: true,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$wanderNeighborConsolesHash();
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return r'wanderNeighborConsolesProvider'
|
||||
''
|
||||
'($argument)';
|
||||
}
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$StreamProviderElement<List<GetNeighborConsolesWithPageCountsResult>>
|
||||
$createElement($ProviderPointer pointer) => $StreamProviderElement(pointer);
|
||||
|
||||
@override
|
||||
Stream<List<GetNeighborConsolesWithPageCountsResult>> create(Ref ref) {
|
||||
final argument = this.argument as Uri;
|
||||
return wanderNeighborConsoles(ref, argument);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is WanderNeighborConsolesProvider &&
|
||||
other.argument == argument;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
return argument.hashCode;
|
||||
}
|
||||
}
|
||||
|
||||
String _$wanderNeighborConsolesHash() =>
|
||||
r'08393800b649cb88e21281dad02a7ed6c745797e';
|
||||
|
||||
final class WanderNeighborConsolesFamily extends $Family
|
||||
with
|
||||
$FunctionalFamilyOverride<
|
||||
Stream<List<GetNeighborConsolesWithPageCountsResult>>,
|
||||
Uri
|
||||
> {
|
||||
WanderNeighborConsolesFamily._()
|
||||
: super(
|
||||
retry: null,
|
||||
name: r'wanderNeighborConsolesProvider',
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
isAutoDispose: true,
|
||||
);
|
||||
|
||||
WanderNeighborConsolesProvider call(Uri consoleUrl) =>
|
||||
WanderNeighborConsolesProvider._(argument: consoleUrl, from: this);
|
||||
|
||||
@override
|
||||
String toString() => r'wanderNeighborConsolesProvider';
|
||||
}
|
||||
|
||||
@ProviderFor(wanderAllConsoles)
|
||||
final wanderAllConsolesProvider = WanderAllConsolesFamily._();
|
||||
|
||||
final class WanderAllConsolesProvider
|
||||
extends
|
||||
$FunctionalProvider<
|
||||
AsyncValue<List<GetAllConsolesWithPageCountsResult>>,
|
||||
List<GetAllConsolesWithPageCountsResult>,
|
||||
Stream<List<GetAllConsolesWithPageCountsResult>>
|
||||
>
|
||||
with
|
||||
$FutureModifier<List<GetAllConsolesWithPageCountsResult>>,
|
||||
$StreamProvider<List<GetAllConsolesWithPageCountsResult>> {
|
||||
WanderAllConsolesProvider._({
|
||||
required WanderAllConsolesFamily super.from,
|
||||
required String super.argument,
|
||||
}) : super(
|
||||
retry: null,
|
||||
name: r'wanderAllConsolesProvider',
|
||||
isAutoDispose: true,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$wanderAllConsolesHash();
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return r'wanderAllConsolesProvider'
|
||||
''
|
||||
'($argument)';
|
||||
}
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$StreamProviderElement<List<GetAllConsolesWithPageCountsResult>>
|
||||
$createElement($ProviderPointer pointer) => $StreamProviderElement(pointer);
|
||||
|
||||
@override
|
||||
Stream<List<GetAllConsolesWithPageCountsResult>> create(Ref ref) {
|
||||
final argument = this.argument as String;
|
||||
return wanderAllConsoles(ref, argument);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is WanderAllConsolesProvider && other.argument == argument;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
return argument.hashCode;
|
||||
}
|
||||
}
|
||||
|
||||
String _$wanderAllConsolesHash() => r'27d87bbad8fb9e993de36c70649b324a98c8bf87';
|
||||
|
||||
final class WanderAllConsolesFamily extends $Family
|
||||
with
|
||||
$FunctionalFamilyOverride<
|
||||
Stream<List<GetAllConsolesWithPageCountsResult>>,
|
||||
String
|
||||
> {
|
||||
WanderAllConsolesFamily._()
|
||||
: super(
|
||||
retry: null,
|
||||
name: r'wanderAllConsolesProvider',
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
isAutoDispose: true,
|
||||
);
|
||||
|
||||
WanderAllConsolesProvider call(String query) =>
|
||||
WanderAllConsolesProvider._(argument: query, from: this);
|
||||
|
||||
@override
|
||||
String toString() => r'wanderAllConsolesProvider';
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:isolate';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:rss_dart/dart_rss.dart';
|
||||
import 'package:uuid/enums.dart';
|
||||
import 'package:weblibre/core/logger.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/extensions/http_encoding.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/database.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_feed_entry.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_small_web_mode.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
|
||||
const _staleDuration = Duration(hours: 3);
|
||||
|
||||
typedef _KagiFeedFetchRequest = ({
|
||||
RootIsolateToken token,
|
||||
String url,
|
||||
String mode,
|
||||
Map<String, String> categoryRemap,
|
||||
});
|
||||
|
||||
class KagiSourceService {
|
||||
final SmallWebDatabase _db;
|
||||
final Map<String, String> _categoryRemap;
|
||||
|
||||
KagiSourceService(this._db, this._categoryRemap);
|
||||
|
||||
Future<bool> needsRefresh(KagiSmallWebMode mode) async {
|
||||
final latestFetch = await _db.smallWebItemDao
|
||||
.getLatestFetchedAt(SmallWebSourceKind.kagi, mode)
|
||||
.getSingleOrNull();
|
||||
|
||||
if (latestFetch == null) return true;
|
||||
|
||||
return DateTime.now().difference(latestFetch) > _staleDuration;
|
||||
}
|
||||
|
||||
Future<void> fetchAndIngest(KagiSmallWebMode mode) async {
|
||||
final request = (
|
||||
token: ServicesBinding.rootIsolateToken!,
|
||||
url: mode.feedUrl.toString(),
|
||||
mode: mode.name,
|
||||
categoryRemap: Map<String, String>.from(_categoryRemap),
|
||||
);
|
||||
|
||||
final List<KagiFeedEntry> entries;
|
||||
try {
|
||||
entries = await _runKagiFeedFetch(request);
|
||||
} catch (e, st) {
|
||||
logger.e(
|
||||
'Failed to fetch/parse Kagi feed for $mode',
|
||||
error: e,
|
||||
stackTrace: st,
|
||||
);
|
||||
rethrow;
|
||||
}
|
||||
|
||||
final now = DateTime.now();
|
||||
await _db.batch((batch) {
|
||||
for (final entry in entries) {
|
||||
final itemId = uuid.v5(Namespace.url.value, entry.url.toString());
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebItems,
|
||||
SmallWebItemsCompanion.insert(
|
||||
id: itemId,
|
||||
url: entry.url,
|
||||
title: Value(entry.title),
|
||||
domain: entry.url.host,
|
||||
author: Value(entry.author),
|
||||
summary: Value(entry.summary),
|
||||
publishedAt: Value(entry.publishedAt),
|
||||
createdAt: Value(now),
|
||||
updatedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebItemsCompanion(
|
||||
title: entry.title != null
|
||||
? Value(entry.title)
|
||||
: const Value.absent(),
|
||||
author: entry.author != null
|
||||
? Value(entry.author)
|
||||
: const Value.absent(),
|
||||
summary: entry.summary != null
|
||||
? Value(entry.summary)
|
||||
: const Value.absent(),
|
||||
publishedAt: entry.publishedAt != null
|
||||
? Value(entry.publishedAt)
|
||||
: const Value.absent(),
|
||||
updatedAt: Value(now),
|
||||
),
|
||||
target: [_db.smallWebItems.url],
|
||||
),
|
||||
);
|
||||
|
||||
final membershipId = uuid.v5(
|
||||
Namespace.url.value,
|
||||
'${SmallWebSourceKind.kagi.name}:${mode.name}:${entry.url}',
|
||||
);
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebMemberships,
|
||||
SmallWebMembershipsCompanion.insert(
|
||||
id: membershipId,
|
||||
itemId: itemId,
|
||||
sourceKind: SmallWebSourceKind.kagi,
|
||||
mode: Value(mode.name),
|
||||
consoleUrl: const Value(null),
|
||||
categories: Value(entry.categories),
|
||||
fetchedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebMembershipsCompanion(
|
||||
categories: Value(entry.categories),
|
||||
fetchedAt: Value(now),
|
||||
),
|
||||
target: [_db.smallWebMemberships.id],
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Future<List<KagiFeedEntry>> _runKagiFeedFetch(_KagiFeedFetchRequest request) {
|
||||
return Isolate.run(_createKagiFeedFetchTask(request));
|
||||
}
|
||||
|
||||
Future<List<KagiFeedEntry>> Function() _createKagiFeedFetchTask(
|
||||
_KagiFeedFetchRequest request,
|
||||
) {
|
||||
return () => _fetchAndParseFeed(
|
||||
request.token,
|
||||
Uri.parse(request.url),
|
||||
request.mode,
|
||||
request.categoryRemap,
|
||||
);
|
||||
}
|
||||
|
||||
Future<List<KagiFeedEntry>> _fetchAndParseFeed(
|
||||
RootIsolateToken token,
|
||||
Uri url,
|
||||
String mode,
|
||||
Map<String, String> categoryRemap,
|
||||
) async {
|
||||
BackgroundIsolateBinaryMessenger.ensureInitialized(token);
|
||||
|
||||
final client = http.Client();
|
||||
try {
|
||||
final response = await client.get(url).timeout(const Duration(seconds: 30));
|
||||
|
||||
if (response.statusCode != 200) {
|
||||
throw Exception(
|
||||
'Kagi feed request failed with status ${response.statusCode}',
|
||||
);
|
||||
}
|
||||
|
||||
final xmlString = response.bodyUnicodeFallback;
|
||||
final feed = AtomFeed.parse(xmlString);
|
||||
|
||||
return feed.items
|
||||
.map((item) {
|
||||
final link = item.links
|
||||
.where((l) => l.rel == 'alternate' || l.rel == null)
|
||||
.map((l) => l.href)
|
||||
.firstOrNull;
|
||||
|
||||
final href = link ?? item.links.firstOrNull?.href;
|
||||
final parsedUrl = href != null ? Uri.tryParse(href) : null;
|
||||
if (parsedUrl == null) return null;
|
||||
|
||||
if (mode == 'videos' && parsedUrl.path.contains('/shorts/')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const kagiScheme = 'https://kagi.com/smallweb/categories';
|
||||
final categories = item.categories
|
||||
.where(
|
||||
(c) =>
|
||||
c.scheme == kagiScheme &&
|
||||
c.term != null &&
|
||||
c.term!.isNotEmpty,
|
||||
)
|
||||
.map((c) => categoryRemap[c.term!] ?? c.term!)
|
||||
.toList();
|
||||
|
||||
final author = item.authors
|
||||
.where((a) => a.name != null && a.name!.isNotEmpty)
|
||||
.map((a) => a.name!)
|
||||
.firstOrNull;
|
||||
|
||||
return KagiFeedEntry(
|
||||
url: parsedUrl,
|
||||
title: item.title,
|
||||
author: author,
|
||||
summary: item.summary,
|
||||
publishedAt: item.updated != null
|
||||
? DateTime.tryParse(item.updated!)
|
||||
: null,
|
||||
categories: categories,
|
||||
);
|
||||
})
|
||||
.nonNulls
|
||||
.toList();
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:math';
|
||||
|
||||
import 'package:fast_equatable/fast_equatable.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/database.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/kagi_small_web_mode.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/wander_console_source.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/kagi_source_service.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/wander_source_service.dart';
|
||||
|
||||
final _random = Random.secure();
|
||||
|
||||
class WanderDiscoverResult with FastEquatable {
|
||||
final SmallWebItem item;
|
||||
final Uri consoleUrl;
|
||||
|
||||
WanderDiscoverResult({required this.item, required this.consoleUrl});
|
||||
|
||||
@override
|
||||
List<Object?> get hashParameters => [item, consoleUrl];
|
||||
}
|
||||
|
||||
class SmallWebDiscoverService {
|
||||
final SmallWebDatabase _db;
|
||||
final KagiSourceService _kagiService;
|
||||
final WanderSourceService _wanderService;
|
||||
|
||||
SmallWebDiscoverService(this._db, this._kagiService, this._wanderService);
|
||||
|
||||
Future<void> recordVisit({
|
||||
required String itemId,
|
||||
required SmallWebSourceKind sourceKind,
|
||||
required KagiSmallWebMode? mode,
|
||||
Uri? consoleUrl,
|
||||
}) async {
|
||||
await _db.smallWebVisitDao.insertVisit(
|
||||
SmallWebVisit(
|
||||
id: uuid.v4(),
|
||||
itemId: itemId,
|
||||
sourceKind: sourceKind,
|
||||
mode: mode?.name,
|
||||
consoleUrl: consoleUrl,
|
||||
visitedAt: DateTime.now(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<SmallWebItem?> discoverKagi({
|
||||
required KagiSmallWebMode mode,
|
||||
String? category,
|
||||
}) async {
|
||||
if (await _kagiService.needsRefresh(mode)) {
|
||||
await _kagiService.fetchAndIngest(mode);
|
||||
}
|
||||
|
||||
final items = await _db.smallWebItemDao
|
||||
.getDiscoverableKagiItems(mode, category)
|
||||
.get();
|
||||
|
||||
if (items.isEmpty) return null;
|
||||
|
||||
final picked = items[_random.nextInt(items.length)];
|
||||
|
||||
await recordVisit(
|
||||
itemId: picked.id,
|
||||
sourceKind: SmallWebSourceKind.kagi,
|
||||
mode: mode,
|
||||
);
|
||||
|
||||
return picked;
|
||||
}
|
||||
|
||||
Future<WanderDiscoverResult?> discoverWander({
|
||||
Uri? currentConsoleUrl,
|
||||
bool forceNewConsole = false,
|
||||
}) async {
|
||||
await _wanderService.syncSeeds();
|
||||
|
||||
final recentItemIds =
|
||||
(await _db.smallWebVisitDao
|
||||
.getRecentItemIds(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
mode: null,
|
||||
)
|
||||
.get())
|
||||
.toSet();
|
||||
|
||||
// Pick a console to explore
|
||||
Uri consoleUrl;
|
||||
if (currentConsoleUrl != null && !forceNewConsole) {
|
||||
consoleUrl = currentConsoleUrl;
|
||||
} else {
|
||||
final consoleUrls = await _wanderService.getDiscoveredConsoleUrls();
|
||||
|
||||
if (forceNewConsole && currentConsoleUrl != null) {
|
||||
consoleUrls.remove(currentConsoleUrl);
|
||||
}
|
||||
|
||||
if (consoleUrls.isEmpty) return null;
|
||||
|
||||
consoleUrl = consoleUrls[_random.nextInt(consoleUrls.length)];
|
||||
}
|
||||
|
||||
final pages = await _refreshAndGetPages(consoleUrl, forceRetry: true);
|
||||
final unvisitedPages = pages
|
||||
.where((page) => !recentItemIds.contains(page.id))
|
||||
.toList();
|
||||
|
||||
if (unvisitedPages.isNotEmpty) {
|
||||
return _pickAndRecord(unvisitedPages, consoleUrl);
|
||||
}
|
||||
|
||||
// No unvisited pages on this console — try alternatives
|
||||
final result = await _tryAlternativeConsoles(consoleUrl, recentItemIds);
|
||||
if (result != null) return result;
|
||||
|
||||
// Last resort: revisit a page from the original console
|
||||
if (pages.isEmpty) return null;
|
||||
return _pickAndRecord(pages, consoleUrl);
|
||||
}
|
||||
|
||||
Future<void> updateItemTitle(String itemId, String title) {
|
||||
return _db.smallWebItemDao.updateTitle(itemId, title);
|
||||
}
|
||||
|
||||
Future<List<SmallWebItem>> _refreshAndGetPages(
|
||||
Uri consoleUrl, {
|
||||
bool forceRetry = false,
|
||||
}) async {
|
||||
if (await _wanderService.shouldRefreshConsole(
|
||||
consoleUrl,
|
||||
forceRetry: forceRetry,
|
||||
)) {
|
||||
await _wanderService.fetchAndIngestConsole(
|
||||
consoleUrl,
|
||||
source: WanderConsoleSource.discovered,
|
||||
);
|
||||
}
|
||||
return _wanderService.getPagesForConsole(consoleUrl);
|
||||
}
|
||||
|
||||
Future<WanderDiscoverResult?> _tryAlternativeConsoles(
|
||||
Uri excludeConsole,
|
||||
Set<String> recentItemIds,
|
||||
) async {
|
||||
final allConsoles = await _wanderService.getDiscoveredConsoleUrls()
|
||||
..remove(excludeConsole)
|
||||
..shuffle(_random);
|
||||
|
||||
for (final altConsole in allConsoles.take(10)) {
|
||||
try {
|
||||
final altPages = await _refreshAndGetPages(altConsole);
|
||||
final unvisited = altPages
|
||||
.where((p) => !recentItemIds.contains(p.id))
|
||||
.toList();
|
||||
final candidates = unvisited.isNotEmpty ? unvisited : altPages;
|
||||
|
||||
if (candidates.isNotEmpty) {
|
||||
return _pickAndRecord(candidates, altConsole);
|
||||
}
|
||||
} catch (_) {
|
||||
// Skip consoles that fail to fetch; continue trying others.
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
Future<WanderDiscoverResult> _pickAndRecord(
|
||||
List<SmallWebItem> candidates,
|
||||
Uri consoleUrl,
|
||||
) async {
|
||||
final picked = candidates[_random.nextInt(candidates.length)];
|
||||
|
||||
await recordVisit(
|
||||
itemId: picked.id,
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
mode: null,
|
||||
consoleUrl: consoleUrl,
|
||||
);
|
||||
|
||||
return WanderDiscoverResult(item: picked, consoleUrl: consoleUrl);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import 'package:weblibre/extensions/uri.dart';
|
||||
|
||||
final _lineCommentPattern = RegExp(r'(?<!:)//.*$', multiLine: true);
|
||||
final _blockCommentPattern = RegExp(r'/\*[\s\S]*?\*/');
|
||||
|
||||
final _consolesPattern = RegExp(r'consoles\s*:\s*\[([\s\S]*?)\]', dotAll: true);
|
||||
final _pagesPattern = RegExp(r'pages\s*:\s*\[([\s\S]*?)\]', dotAll: true);
|
||||
// ignore: unnecessary_raw_strings
|
||||
final _stringPattern = RegExp(r'''(?:["'`])([^"'`]+)(?:["'`])''');
|
||||
|
||||
List<String> _extractArray(String source, RegExp pattern) {
|
||||
final match = pattern.firstMatch(source);
|
||||
if (match == null) return [];
|
||||
|
||||
final arrayContent = match.group(1) ?? '';
|
||||
return _stringPattern
|
||||
.allMatches(arrayContent)
|
||||
.map((m) => m.group(1)!)
|
||||
.toList();
|
||||
}
|
||||
|
||||
Uri _normalizeUrl(String url) {
|
||||
var normalized = url;
|
||||
|
||||
if (normalized.endsWith('/index.html')) {
|
||||
normalized = normalized.substring(
|
||||
0,
|
||||
normalized.length - 'index.html'.length,
|
||||
);
|
||||
}
|
||||
|
||||
return Uri.parse(normalized);
|
||||
}
|
||||
|
||||
class WanderJsResult {
|
||||
final List<Uri> consoles;
|
||||
final List<Uri> pages;
|
||||
|
||||
const WanderJsResult({required this.consoles, required this.pages});
|
||||
|
||||
factory WanderJsResult.parse(String jsSource) {
|
||||
final cleaned = jsSource
|
||||
.replaceAll(_blockCommentPattern, '')
|
||||
.replaceAll(_lineCommentPattern, '');
|
||||
|
||||
final consoles = _extractArray(cleaned, _consolesPattern);
|
||||
final pages = _extractArray(cleaned, _pagesPattern);
|
||||
|
||||
return WanderJsResult(
|
||||
consoles: consoles
|
||||
.map(_normalizeUrl)
|
||||
.where((uri) => uri.isHttpOrHttps)
|
||||
.toList(),
|
||||
pages: pages
|
||||
.map(_normalizeUrl)
|
||||
.where((uri) => uri.isHttpOrHttps)
|
||||
.toList(),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:isolate';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:uuid/enums.dart';
|
||||
import 'package:weblibre/core/logger.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/database.dart';
|
||||
import 'package:weblibre/features/small_web/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/small_web_source_kind.dart';
|
||||
import 'package:weblibre/features/small_web/data/models/wander_console_source.dart';
|
||||
import 'package:weblibre/features/small_web/data/wander_seed_consoles.dart';
|
||||
import 'package:weblibre/features/small_web/domain/services/wander_js_parser.dart';
|
||||
|
||||
const _staleDuration = Duration(hours: 3);
|
||||
const _retryAfterError = Duration(minutes: 30);
|
||||
|
||||
typedef _WanderJsFetchRequest = ({RootIsolateToken token, String url});
|
||||
|
||||
class WanderSourceService {
|
||||
final SmallWebDatabase _db;
|
||||
|
||||
WanderSourceService(this._db);
|
||||
|
||||
Future<bool> shouldRefreshConsole(
|
||||
Uri consoleUrl, {
|
||||
bool forceRetry = false,
|
||||
}) async {
|
||||
final console = await _db.wanderConsoleDao
|
||||
.getConsole(consoleUrl)
|
||||
.getSingleOrNull();
|
||||
|
||||
if (console == null || console.lastFetchedAt == null) return true;
|
||||
|
||||
final age = DateTime.now().difference(console.lastFetchedAt!);
|
||||
if (console.lastFetchFailed == true) {
|
||||
return forceRetry || age > _retryAfterError;
|
||||
}
|
||||
|
||||
return age > _staleDuration;
|
||||
}
|
||||
|
||||
Future<void> syncSeeds() async {
|
||||
final now = DateTime.now();
|
||||
await _db.batch((batch) {
|
||||
for (final seedUrl in wanderSeedConsoles) {
|
||||
final url = Uri.parse(seedUrl);
|
||||
batch.insert(
|
||||
_db.wanderConsoles,
|
||||
WanderConsolesCompanion.insert(
|
||||
url: url,
|
||||
wanderJsUrl: url.resolve('wander.js'),
|
||||
source: WanderConsoleSource.seed,
|
||||
createdAt: Value(now),
|
||||
),
|
||||
onConflict: DoNothing(),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> fetchAndIngestConsole(
|
||||
Uri consoleUrl, {
|
||||
required WanderConsoleSource source,
|
||||
}) async {
|
||||
final wanderJsUrl = consoleUrl.resolve('wander.js');
|
||||
final now = DateTime.now();
|
||||
|
||||
try {
|
||||
final result = await _runWanderJsFetch((
|
||||
token: ServicesBinding.rootIsolateToken!,
|
||||
url: wanderJsUrl.toString(),
|
||||
));
|
||||
|
||||
if (result == null) {
|
||||
await _saveConsoleWithError(consoleUrl, wanderJsUrl, now, source);
|
||||
return null;
|
||||
}
|
||||
|
||||
final existingUrls = await _db.wanderConsoleDao
|
||||
.getExistingConsoleUrls(result.consoles)
|
||||
.get();
|
||||
|
||||
await _db.transaction(() async {
|
||||
await _db.wanderConsoleDao.upsertConsole(
|
||||
WanderConsole(
|
||||
url: consoleUrl,
|
||||
wanderJsUrl: wanderJsUrl,
|
||||
lastFetchedAt: now,
|
||||
lastFetchFailed: false,
|
||||
source: source,
|
||||
createdAt: now,
|
||||
),
|
||||
);
|
||||
|
||||
await _db.batch((batch) {
|
||||
for (final neighborUrl in result.consoles) {
|
||||
batch.insert(
|
||||
_db.wanderConsoleNeighbors,
|
||||
WanderConsoleNeighborsCompanion.insert(
|
||||
sourceConsoleUrl: consoleUrl.toString(),
|
||||
targetConsoleUrl: neighborUrl.toString(),
|
||||
discoveredAt: Value(now),
|
||||
),
|
||||
onConflict: DoNothing(),
|
||||
);
|
||||
|
||||
if (!existingUrls.contains(neighborUrl)) {
|
||||
batch.insert(
|
||||
_db.wanderConsoles,
|
||||
WanderConsolesCompanion.insert(
|
||||
url: neighborUrl,
|
||||
wanderJsUrl: neighborUrl.resolve('wander.js'),
|
||||
discoveredFromUrl: Value(consoleUrl),
|
||||
source: WanderConsoleSource.discovered,
|
||||
createdAt: Value(now),
|
||||
),
|
||||
onConflict: DoNothing(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (final pageUrl in result.pages) {
|
||||
final itemId = uuid.v5(Namespace.url.value, pageUrl.toString());
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebItems,
|
||||
SmallWebItemsCompanion.insert(
|
||||
id: itemId,
|
||||
url: pageUrl,
|
||||
domain: pageUrl.host,
|
||||
createdAt: Value(now),
|
||||
updatedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebItemsCompanion(updatedAt: Value(now)),
|
||||
target: [_db.smallWebItems.url],
|
||||
),
|
||||
);
|
||||
|
||||
final membershipId = uuid.v5(
|
||||
Namespace.url.value,
|
||||
'${SmallWebSourceKind.wander.name}:$consoleUrl:$pageUrl',
|
||||
);
|
||||
|
||||
batch.insert(
|
||||
_db.smallWebMemberships,
|
||||
SmallWebMembershipsCompanion.insert(
|
||||
id: membershipId,
|
||||
itemId: itemId,
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
consoleUrl: Value(consoleUrl),
|
||||
fetchedAt: Value(now),
|
||||
),
|
||||
onConflict: DoUpdate(
|
||||
(old) => SmallWebMembershipsCompanion(fetchedAt: Value(now)),
|
||||
target: [_db.smallWebMemberships.id],
|
||||
),
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (e, st) {
|
||||
logger.e(
|
||||
'Failed to fetch wander.js from $wanderJsUrl',
|
||||
error: e,
|
||||
stackTrace: st,
|
||||
);
|
||||
await _saveConsoleWithError(consoleUrl, wanderJsUrl, now, source);
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
/// Normalizes a user-input URL to a wander console URL.
|
||||
///
|
||||
/// Accepts URLs like:
|
||||
/// - `https://example.com/wander/` → kept as-is
|
||||
/// - `https://example.com/wander` → trailing slash added
|
||||
/// - `https://example.com` → `/wander/` appended
|
||||
/// - `https://example.com/` → `wander/` appended
|
||||
///
|
||||
/// Returns the normalized console URL (always ends with `/wander/`).
|
||||
static Uri normalizeConsoleUrl(Uri url) {
|
||||
var path = url.path;
|
||||
|
||||
// Strip trailing wander.js if someone pasted the full JS URL
|
||||
if (path.endsWith('/wander.js')) {
|
||||
path = path.substring(0, path.length - 'wander.js'.length);
|
||||
}
|
||||
|
||||
// Ensure the path ends with /wander/
|
||||
if (!path.endsWith('/wander/')) {
|
||||
if (path.endsWith('/wander')) {
|
||||
path = '$path/';
|
||||
} else {
|
||||
if (!path.endsWith('/')) {
|
||||
path = '$path/';
|
||||
}
|
||||
path = '${path}wander/';
|
||||
}
|
||||
}
|
||||
|
||||
return url.replace(path: path);
|
||||
}
|
||||
|
||||
/// Checks if a console URL already exists in the database.
|
||||
Future<bool> consoleExists(Uri consoleUrl) async {
|
||||
final console = await _db.wanderConsoleDao
|
||||
.getConsole(consoleUrl)
|
||||
.getSingleOrNull();
|
||||
|
||||
return console != null;
|
||||
}
|
||||
|
||||
/// Validates that a URL points to a valid wander console by fetching its
|
||||
/// wander.js and checking it contains valid consoles or pages data.
|
||||
///
|
||||
/// Returns the parsed [WanderJsResult] if valid, or throws with a
|
||||
/// descriptive error message.
|
||||
Future<WanderJsResult> validateConsole(Uri consoleUrl) async {
|
||||
final wanderJsUrl = consoleUrl.resolve('wander.js');
|
||||
|
||||
final result = await _runWanderJsFetch((
|
||||
token: ServicesBinding.rootIsolateToken!,
|
||||
url: wanderJsUrl.toString(),
|
||||
));
|
||||
|
||||
if (result == null) {
|
||||
throw Exception('Could not fetch wander.js from $wanderJsUrl');
|
||||
}
|
||||
|
||||
if (result.consoles.isEmpty && result.pages.isEmpty) {
|
||||
throw Exception('The wander.js file contains no consoles or pages');
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Validates and adds a user-provided console URL.
|
||||
///
|
||||
/// The URL is normalized, checked for duplicates, validated by fetching
|
||||
/// wander.js, then ingested into the database.
|
||||
///
|
||||
/// Returns the normalized console URL.
|
||||
Future<Uri> addConsoleFromUrl(Uri rawUrl) async {
|
||||
final consoleUrl = normalizeConsoleUrl(rawUrl);
|
||||
|
||||
if (await consoleExists(consoleUrl)) {
|
||||
throw Exception('This console has already been added');
|
||||
}
|
||||
|
||||
// Validate by fetching wander.js
|
||||
await validateConsole(consoleUrl);
|
||||
|
||||
// Now do the full ingest
|
||||
await fetchAndIngestConsole(consoleUrl, source: WanderConsoleSource.manual);
|
||||
|
||||
return consoleUrl;
|
||||
}
|
||||
|
||||
Future<List<Uri>> getDiscoveredConsoleUrls() {
|
||||
return _db.wanderConsoleDao.getDiscoveredConsoleUrls().get();
|
||||
}
|
||||
|
||||
Future<List<SmallWebItem>> getPagesForConsole(Uri consoleUrl) {
|
||||
return _db.definitionsDrift
|
||||
.getWanderPagesForConsole(
|
||||
sourceKind: SmallWebSourceKind.wander,
|
||||
consoleUrl: consoleUrl.toString(),
|
||||
)
|
||||
.get();
|
||||
}
|
||||
|
||||
Future<void> _saveConsoleWithError(
|
||||
Uri consoleUrl,
|
||||
Uri wanderJsUrl,
|
||||
DateTime now,
|
||||
WanderConsoleSource source,
|
||||
) async {
|
||||
await _db.wanderConsoleDao.upsertConsole(
|
||||
WanderConsole(
|
||||
url: consoleUrl,
|
||||
wanderJsUrl: wanderJsUrl,
|
||||
lastFetchedAt: now,
|
||||
lastFetchFailed: true,
|
||||
source: source,
|
||||
createdAt: now,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> _runWanderJsFetch(_WanderJsFetchRequest request) {
|
||||
return Isolate.run(_createWanderJsFetchTask(request));
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> Function() _createWanderJsFetchTask(
|
||||
_WanderJsFetchRequest request,
|
||||
) {
|
||||
return () => _fetchAndParseWanderJs(request.token, Uri.parse(request.url));
|
||||
}
|
||||
|
||||
Future<WanderJsResult?> _fetchAndParseWanderJs(
|
||||
RootIsolateToken token,
|
||||
Uri url,
|
||||
) async {
|
||||
BackgroundIsolateBinaryMessenger.ensureInitialized(token);
|
||||
|
||||
final client = http.Client();
|
||||
try {
|
||||
final response = await client.get(url).timeout(const Duration(seconds: 15));
|
||||
|
||||
if (response.statusCode != 200) return null;
|
||||
|
||||
return WanderJsResult.parse(response.body);
|
||||
} finally {
|
||||
client.close();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user