From 70d1c1b50ef9bd9572a9ecebd8895c43c94770fc Mon Sep 17 00:00:00 2001 From: Fabian Freund Date: Wed, 5 Mar 2025 14:27:13 +0100 Subject: [PATCH] background fetch running --- .vscode/settings.json | 3 + app/android/build.gradle | 1 + app/lib/features/bangs/data/providers.dart | 6 +- app/lib/features/bangs/data/providers.g.dart | 2 +- .../features/chat_store/data/providers.dart | 6 +- .../features/chat_store/data/providers.g.dart | 2 +- .../domain/providers/selected_tab.dart | 6 +- .../domain/providers/selected_tab.g.dart | 2 +- .../geckoview/domain/providers/tab_list.dart | 6 +- .../domain/providers/tab_list.g.dart | 2 +- .../geckoview/domain/providers/tab_state.dart | 4 +- .../domain/providers/tab_state.g.dart | 2 +- .../providers/web_extensions_state.dart | 4 +- .../providers/web_extensions_state.g.dart | 2 +- .../geckoview/domain/repositories/tab.dart | 6 +- .../geckoview/domain/repositories/tab.g.dart | 2 +- .../features/browser/domain/providers.dart | 6 +- .../features/browser/domain/providers.g.dart | 2 +- ...n.dart => proxy_settings_replication.dart} | 4 +- ...dart => proxy_settings_replication.g.dart} | 22 +- .../widgets/browser_modules/browser_view.dart | 10 +- .../repositories/preference_settings.dart | 4 +- .../repositories/preference_settings.g.dart | 2 +- .../search/presentation/screens/search.dart | 2 + .../widgets/search_modules/feed_search.dart | 171 ++++++++++++ .../search_modules/history_suggestions.dart | 20 +- .../features/tabs/data/providers.dart | 6 +- .../features/tabs/data/providers.g.dart | 2 +- .../domain/repositories/document.dart | 4 +- .../domain/repositories/document.g.dart | 2 +- .../tor/domain/services/tor_proxy.dart | 4 +- .../tor/domain/services/tor_proxy.g.dart | 2 +- app/lib/features/user/data/providers.dart | 6 +- app/lib/features/user/data/providers.g.dart | 2 +- .../domain/repositories/engine_settings.dart | 4 +- .../repositories/engine_settings.g.dart | 2 +- .../domain/repositories/general_settings.dart | 4 +- .../repositories/general_settings.g.dart | 2 +- .../web_feed/data/database/daos/article.dart | 37 +++ .../web_feed/data/database/database.drift | 5 +- .../web_feed/data/database/database.g.dart | 116 ++++++++ .../web_feed/data/models/feed_article.dart | 17 ++ .../web_feed/data/models/feed_article.g.dart | 255 ++++++++++++++++++ .../models/feed_article_query_result.dart | 2 + app/lib/features/web_feed/data/providers.dart | 5 +- .../features/web_feed/data/providers.g.dart | 2 +- .../features/web_feed/domain/providers.dart | 3 +- .../features/web_feed/domain/providers.g.dart | 2 +- .../services/article_content_processor.dart | 51 ++++ .../services/article_content_processor.g.dart | 28 ++ .../web_feed/domain/services/feed_reader.dart | 2 +- .../domain/services/feed_reader.g.dart | 2 +- .../controllers/fetch_articles.dart | 27 +- .../controllers/fetch_articles.g.dart | 2 +- .../widgets/feed_article_card.dart | 9 +- .../features/web_feed/utils/feed_parser.dart | 81 ++---- .../web_feed/utils/fetch_entrypoint.dart | 35 +++ app/lib/main.dart | 33 ++- app/pubspec.yaml | 1 + .../lib/flutter_mozilla_components.dart | 2 +- .../services/gecko_browser_extension.dart | 13 +- .../.plugin_symlinks/sqlite3_flutter_libs | 2 +- 62 files changed, 898 insertions(+), 173 deletions(-) create mode 100644 .vscode/settings.json rename app/lib/features/geckoview/features/browser/domain/services/{proxy_settings_replucation.dart => proxy_settings_replication.dart} (91%) rename app/lib/features/geckoview/features/browser/domain/services/{proxy_settings_replucation.g.dart => proxy_settings_replication.g.dart} (54%) create mode 100644 app/lib/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart create mode 100644 app/lib/features/web_feed/domain/services/article_content_processor.dart create mode 100644 app/lib/features/web_feed/domain/services/article_content_processor.g.dart create mode 100644 app/lib/features/web_feed/utils/fetch_entrypoint.dart diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..7d1a672c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "dart.runPubGetOnPubspecChanges": "never" +} \ No newline at end of file diff --git a/app/android/build.gradle b/app/android/build.gradle index 3eef6e9c..16b200b0 100644 --- a/app/android/build.gradle +++ b/app/android/build.gradle @@ -6,6 +6,7 @@ allprojects { repositories { google() mavenCentral() + maven { url "${project(':background_fetch').projectDir}/libs" } } } diff --git a/app/lib/features/bangs/data/providers.dart b/app/lib/features/bangs/data/providers.dart index c7cf98ee..2fc62514 100644 --- a/app/lib/features/bangs/data/providers.dart +++ b/app/lib/features/bangs/data/providers.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:drift/drift.dart'; import 'package:drift/native.dart'; import 'package:lensai/features/bangs/data/database/database.dart'; @@ -38,8 +36,8 @@ BangDatabase bangDatabase(Ref ref) { }), ); - ref.onDispose(() { - unawaited(db.close()); + ref.onDispose(() async { + await db.close(); }); return db; diff --git a/app/lib/features/bangs/data/providers.g.dart b/app/lib/features/bangs/data/providers.g.dart index 0ba9f339..c007ae1c 100644 --- a/app/lib/features/bangs/data/providers.g.dart +++ b/app/lib/features/bangs/data/providers.g.dart @@ -6,7 +6,7 @@ part of 'providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$bangDatabaseHash() => r'1ff3fb8bfa29269f6e515f3b22c216998769f578'; +String _$bangDatabaseHash() => r'e31cf4b499300aff65174a16fb74c7dd8ec3d6f0'; /// See also [bangDatabase]. @ProviderFor(bangDatabase) diff --git a/app/lib/features/chat/features/chat_store/data/providers.dart b/app/lib/features/chat/features/chat_store/data/providers.dart index afe2a95b..c0f897f8 100644 --- a/app/lib/features/chat/features/chat_store/data/providers.dart +++ b/app/lib/features/chat/features/chat_store/data/providers.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:drift/drift.dart'; import 'package:drift/native.dart'; import 'package:lensai/data/database/functions/lexo_rank_functions.dart'; @@ -44,8 +42,8 @@ ChatDatabase chatDatabase(Ref ref) { }), ); - ref.onDispose(() { - unawaited(db.close()); + ref.onDispose(() async { + await db.close(); }); return db; diff --git a/app/lib/features/chat/features/chat_store/data/providers.g.dart b/app/lib/features/chat/features/chat_store/data/providers.g.dart index 70a8d3b1..c5afff20 100644 --- a/app/lib/features/chat/features/chat_store/data/providers.g.dart +++ b/app/lib/features/chat/features/chat_store/data/providers.g.dart @@ -6,7 +6,7 @@ part of 'providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$chatDatabaseHash() => r'b48955bc016f8c74684ed6e2c5cba99f5ab38dfe'; +String _$chatDatabaseHash() => r'd07fef426c6984e6fb2eab9651a5c51aaca98961'; /// See also [chatDatabase]. @ProviderFor(chatDatabase) diff --git a/app/lib/features/geckoview/domain/providers/selected_tab.dart b/app/lib/features/geckoview/domain/providers/selected_tab.dart index 6cd3f697..ee382b2a 100644 --- a/app/lib/features/geckoview/domain/providers/selected_tab.dart +++ b/app/lib/features/geckoview/domain/providers/selected_tab.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; import 'package:lensai/features/geckoview/domain/providers.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; @@ -25,8 +23,8 @@ class SelectedTab extends _$SelectedTab { state = tabId; }); - ref.onDispose(() { - unawaited(selectedTabSub.cancel()); + ref.onDispose(() async { + await selectedTabSub.cancel(); }); return null; diff --git a/app/lib/features/geckoview/domain/providers/selected_tab.g.dart b/app/lib/features/geckoview/domain/providers/selected_tab.g.dart index 11c0cc03..dd083f3b 100644 --- a/app/lib/features/geckoview/domain/providers/selected_tab.g.dart +++ b/app/lib/features/geckoview/domain/providers/selected_tab.g.dart @@ -6,7 +6,7 @@ part of 'selected_tab.dart'; // RiverpodGenerator // ************************************************************************** -String _$selectedTabHash() => r'250f54fe9a56d14876e0979e5370eb76879da11c'; +String _$selectedTabHash() => r'12b670dd76c3416e8d62bafd63bf69a83a603e3a'; /// See also [SelectedTab]. @ProviderFor(SelectedTab) diff --git a/app/lib/features/geckoview/domain/providers/tab_list.dart b/app/lib/features/geckoview/domain/providers/tab_list.dart index 30379737..691f4d7d 100644 --- a/app/lib/features/geckoview/domain/providers/tab_list.dart +++ b/app/lib/features/geckoview/domain/providers/tab_list.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:fast_equatable/fast_equatable.dart'; import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; import 'package:lensai/features/geckoview/domain/providers.dart'; @@ -30,8 +28,8 @@ class TabList extends _$TabList { } }); - ref.onDispose(() { - unawaited(tabListSub.cancel()); + ref.onDispose(() async { + await tabListSub.cancel(); }); return EquatableValue([]); diff --git a/app/lib/features/geckoview/domain/providers/tab_list.g.dart b/app/lib/features/geckoview/domain/providers/tab_list.g.dart index 3099083f..866dc819 100644 --- a/app/lib/features/geckoview/domain/providers/tab_list.g.dart +++ b/app/lib/features/geckoview/domain/providers/tab_list.g.dart @@ -6,7 +6,7 @@ part of 'tab_list.dart'; // RiverpodGenerator // ************************************************************************** -String _$tabListHash() => r'2b98ccdf62c15670e0a12e38064e8186e6555cf8'; +String _$tabListHash() => r'9507bcc75dc69a55aba14334cdef0128a5fcf252'; /// See also [TabList]. @ProviderFor(TabList) diff --git a/app/lib/features/geckoview/domain/providers/tab_state.dart b/app/lib/features/geckoview/domain/providers/tab_state.dart index d924239f..b505941f 100644 --- a/app/lib/features/geckoview/domain/providers/tab_state.dart +++ b/app/lib/features/geckoview/domain/providers/tab_state.dart @@ -165,9 +165,9 @@ class TabStates extends _$TabStates { } }); - ref.onDispose(() { + ref.onDispose(() async { for (final sub in subscriptions) { - unawaited(sub.cancel()); + await sub.cancel(); } }); diff --git a/app/lib/features/geckoview/domain/providers/tab_state.g.dart b/app/lib/features/geckoview/domain/providers/tab_state.g.dart index 261f0300..af7b62f4 100644 --- a/app/lib/features/geckoview/domain/providers/tab_state.g.dart +++ b/app/lib/features/geckoview/domain/providers/tab_state.g.dart @@ -160,7 +160,7 @@ final selectedTabStateProvider = AutoDisposeProvider.internal( @Deprecated('Will be removed in 3.0. Use Ref instead') // ignore: unused_element typedef SelectedTabStateRef = AutoDisposeProviderRef; -String _$tabStatesHash() => r'b136dbf6d68b67f0fe0db302a4147b67789db853'; +String _$tabStatesHash() => r'3a906aec8c4e650de8d5d2461ff45a99b8a0a3b9'; /// See also [TabStates]. @ProviderFor(TabStates) diff --git a/app/lib/features/geckoview/domain/providers/web_extensions_state.dart b/app/lib/features/geckoview/domain/providers/web_extensions_state.dart index 1f7b7b0c..927ff318 100644 --- a/app/lib/features/geckoview/domain/providers/web_extensions_state.dart +++ b/app/lib/features/geckoview/domain/providers/web_extensions_state.dart @@ -86,9 +86,9 @@ class WebExtensionsState extends _$WebExtensionsState { ], }; - ref.onDispose(() { + ref.onDispose(() async { for (final sub in subscriptions) { - unawaited(sub.cancel()); + await sub.cancel(); } }); diff --git a/app/lib/features/geckoview/domain/providers/web_extensions_state.g.dart b/app/lib/features/geckoview/domain/providers/web_extensions_state.g.dart index 36b62dbd..d9893c47 100644 --- a/app/lib/features/geckoview/domain/providers/web_extensions_state.g.dart +++ b/app/lib/features/geckoview/domain/providers/web_extensions_state.g.dart @@ -7,7 +7,7 @@ part of 'web_extensions_state.dart'; // ************************************************************************** String _$webExtensionsStateHash() => - r'eab924e9c17e805a3c29065289be5d1c8ffe3bc9'; + r'6fb44ab543250f2b2554e410ee749fdd035cd4e7'; /// Copied from Dart SDK class _SystemHash { diff --git a/app/lib/features/geckoview/domain/repositories/tab.dart b/app/lib/features/geckoview/domain/repositories/tab.dart index b61a0fe6..31b32f03 100644 --- a/app/lib/features/geckoview/domain/repositories/tab.dart +++ b/app/lib/features/geckoview/domain/repositories/tab.dart @@ -167,10 +167,10 @@ class TabRepository extends _$TabRepository { }); }); - ref.onDispose(() { + ref.onDispose(() async { tabStateDebouncer.dispose(); - unawaited(tabAddedSub.cancel()); - unawaited(tabContentSub.cancel()); + await tabAddedSub.cancel(); + await tabContentSub.cancel(); }); } } diff --git a/app/lib/features/geckoview/domain/repositories/tab.g.dart b/app/lib/features/geckoview/domain/repositories/tab.g.dart index 6240102e..faed5375 100644 --- a/app/lib/features/geckoview/domain/repositories/tab.g.dart +++ b/app/lib/features/geckoview/domain/repositories/tab.g.dart @@ -6,7 +6,7 @@ part of 'tab.dart'; // RiverpodGenerator // ************************************************************************** -String _$tabRepositoryHash() => r'69e283b820d86388f4d80e71e08761e2b6890507'; +String _$tabRepositoryHash() => r'35c078a841d815aa2452c07b52c4cf0cbcec7e68'; /// See also [TabRepository]. @ProviderFor(TabRepository) diff --git a/app/lib/features/geckoview/features/browser/domain/providers.dart b/app/lib/features/geckoview/features/browser/domain/providers.dart index d4e6b941..1b4937d2 100644 --- a/app/lib/features/geckoview/features/browser/domain/providers.dart +++ b/app/lib/features/geckoview/features/browser/domain/providers.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:fast_equatable/fast_equatable.dart'; import 'package:lensai/features/bangs/data/models/bang_data.dart'; import 'package:lensai/features/bangs/domain/repositories/data.dart'; @@ -48,8 +46,8 @@ class SelectedBangData extends _$SelectedBangData { state = value; }); - ref.onDispose(() { - unawaited(subscription.cancel()); + ref.onDispose(() async { + await subscription.cancel(); }); return null; diff --git a/app/lib/features/geckoview/features/browser/domain/providers.g.dart b/app/lib/features/geckoview/features/browser/domain/providers.g.dart index 0000cbb7..874b8321 100644 --- a/app/lib/features/geckoview/features/browser/domain/providers.g.dart +++ b/app/lib/features/geckoview/features/browser/domain/providers.g.dart @@ -719,7 +719,7 @@ class _SelectedBangTriggerProviderElement String? get domain => (origin as SelectedBangTriggerProvider).domain; } -String _$selectedBangDataHash() => r'28c394e5989e108870ca5733e916b80b3cf8972c'; +String _$selectedBangDataHash() => r'666858556efe59d2de38a04fda30e8ec227c6675'; abstract class _$SelectedBangData extends BuildlessAutoDisposeNotifier { diff --git a/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replucation.dart b/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replication.dart similarity index 91% rename from app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replucation.dart rename to app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replication.dart index ac80953a..c91e7938 100644 --- a/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replucation.dart +++ b/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replication.dart @@ -4,10 +4,10 @@ import 'package:lensai/features/geckoview/features/tabs/domain/providers.dart'; import 'package:lensai/features/tor/domain/services/tor_proxy.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; -part 'proxy_settings_replucation.g.dart'; +part 'proxy_settings_replication.g.dart'; @Riverpod(keepAlive: true) -class ProxySettingsReplucation extends _$ProxySettingsReplucation { +class ProxySettingsReplication extends _$ProxySettingsReplication { final _service = GeckoContainerProxyService(); @override diff --git a/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replucation.g.dart b/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replication.g.dart similarity index 54% rename from app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replucation.g.dart rename to app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replication.g.dart index ec8f2422..4a328b21 100644 --- a/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replucation.g.dart +++ b/app/lib/features/geckoview/features/browser/domain/services/proxy_settings_replication.g.dart @@ -1,28 +1,28 @@ // GENERATED CODE - DO NOT MODIFY BY HAND -part of 'proxy_settings_replucation.dart'; +part of 'proxy_settings_replication.dart'; // ************************************************************************** // RiverpodGenerator // ************************************************************************** -String _$proxySettingsReplucationHash() => - r'b00fc9a9c8b385a0f8a5830e9ba1c329d340c0ae'; +String _$proxySettingsReplicationHash() => + r'1982923eeb0bcd6d606683fde70835ec7aff601b'; -/// See also [ProxySettingsReplucation]. -@ProviderFor(ProxySettingsReplucation) -final proxySettingsReplucationProvider = - NotifierProvider.internal( - ProxySettingsReplucation.new, - name: r'proxySettingsReplucationProvider', +/// See also [ProxySettingsReplication]. +@ProviderFor(ProxySettingsReplication) +final proxySettingsReplicationProvider = + NotifierProvider.internal( + ProxySettingsReplication.new, + name: r'proxySettingsReplicationProvider', debugGetCreateSourceHash: const bool.fromEnvironment('dart.vm.product') ? null - : _$proxySettingsReplucationHash, + : _$proxySettingsReplicationHash, dependencies: null, allTransitiveDependencies: null, ); -typedef _$ProxySettingsReplucation = Notifier; +typedef _$ProxySettingsReplication = Notifier; // ignore_for_file: type=lint // ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart b/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart index 261a3b69..402c3480 100644 --- a/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart +++ b/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart @@ -15,11 +15,12 @@ import 'package:lensai/features/geckoview/domain/providers/web_extensions_state. import 'package:lensai/features/geckoview/domain/repositories/tab.dart'; import 'package:lensai/features/geckoview/features/browser/domain/services/browser_data.dart'; import 'package:lensai/features/geckoview/features/browser/domain/services/engine_settings_replication.dart'; -import 'package:lensai/features/geckoview/features/browser/domain/services/proxy_settings_replucation.dart'; +import 'package:lensai/features/geckoview/features/browser/domain/services/proxy_settings_replication.dart'; import 'package:lensai/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart'; import 'package:lensai/features/user/domain/repositories/cache.dart'; import 'package:lensai/features/user/domain/repositories/general_settings.dart'; import 'package:lensai/features/user/domain/services/local_authentication.dart'; +import 'package:lensai/features/web_feed/domain/services/article_content_processor.dart'; class BrowserView extends StatefulHookConsumerWidget { final Duration screenshotPeriod; @@ -151,7 +152,12 @@ class _BrowserViewState extends ConsumerState (previous, next) {}, ); - ref.listenManual(proxySettingsReplucationProvider, (previous, next) {}); + ref.listenManual(proxySettingsReplicationProvider, (previous, next) {}); + + ref.listenManual( + articleContentProcessorServiceProvider, + (previous, next) {}, + ); } @override diff --git a/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.dart b/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.dart index f43eb68b..c04b5259 100644 --- a/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.dart +++ b/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.dart @@ -68,8 +68,8 @@ class _PreferenceRepository extends _$PreferenceRepository { Raw>> build() { _prefSubject = BehaviorSubject>(); - ref.onDispose(() { - unawaited(_prefSubject.close()); + ref.onDispose(() async { + await _prefSubject.close(); }); ref.onAddListener(() async { diff --git a/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.g.dart b/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.g.dart index 203199da..8d05205a 100644 --- a/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.g.dart +++ b/app/lib/features/geckoview/features/preferences/data/repositories/preference_settings.g.dart @@ -330,7 +330,7 @@ class _PreferenceSettingGroupProviderElement } String _$preferenceRepositoryHash() => - r'53c0cbbf60e44c3834fe3e3255836f7e27f09c7f'; + r'b2b7f75b3d30842b3006bb18af89f7ff90ba4ce7'; /// See also [_PreferenceRepository]. @ProviderFor(_PreferenceRepository) diff --git a/app/lib/features/geckoview/features/search/presentation/screens/search.dart b/app/lib/features/geckoview/features/search/presentation/screens/search.dart index 4fd56cad..4b70f234 100644 --- a/app/lib/features/geckoview/features/search/presentation/screens/search.dart +++ b/app/lib/features/geckoview/features/search/presentation/screens/search.dart @@ -9,6 +9,7 @@ import 'package:lensai/features/geckoview/domain/controllers/bottom_sheet.dart'; import 'package:lensai/features/geckoview/domain/repositories/tab.dart'; import 'package:lensai/features/geckoview/features/browser/domain/providers.dart'; import 'package:lensai/features/geckoview/features/search/presentation/widgets/search_field.dart'; +import 'package:lensai/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart'; import 'package:lensai/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart'; import 'package:lensai/features/geckoview/features/search/presentation/widgets/search_modules/search_suggestions.dart'; import 'package:lensai/features/geckoview/features/search/presentation/widgets/search_modules/tab_search.dart'; @@ -116,6 +117,7 @@ class SearchScreen extends HookConsumerWidget { ), const SliverToBoxAdapter(child: Divider()), TabSearch(searchTextController: searchTextController), + FeedSearch(searchTextController: searchTextController), HistorySuggestions(searchTextController: searchTextController), ], ), diff --git a/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart b/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart new file mode 100644 index 00000000..a3a3d0ee --- /dev/null +++ b/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart @@ -0,0 +1,171 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_markdown/flutter_markdown.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:lensai/core/routing/routes.dart'; +import 'package:lensai/extensions/nullable.dart'; +import 'package:lensai/extensions/uri.dart'; +import 'package:lensai/features/web_feed/data/models/feed_article_query_result.dart'; +import 'package:lensai/features/web_feed/data/models/feed_link.dart'; +import 'package:lensai/features/web_feed/domain/providers.dart'; +import 'package:lensai/features/web_feed/extensions/atom.dart'; +import 'package:lensai/features/web_feed/extensions/feed_article.dart'; +import 'package:lensai/presentation/hooks/listenable_callback.dart'; +import 'package:lensai/presentation/widgets/failure_widget.dart'; +import 'package:lensai/presentation/widgets/url_icon.dart'; +import 'package:skeletonizer/skeletonizer.dart'; +import 'package:sliver_tools/sliver_tools.dart'; + +class FeedSearch extends HookConsumerWidget { + static const _matchPrefix = '***'; + static const _matchSuffix = '***'; + + final TextEditingController searchTextController; + + const FeedSearch({required this.searchTextController}); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final theme = Theme.of(context); + + final articlesAsync = ref.watch(articleSearchProvider(null)); + + useListenableCallback(searchTextController, () async { + await ref + .read(articleSearchProvider(null).notifier) + .search( + searchTextController.text, + // ignore: avoid_redundant_argument_values dont break things + matchPrefix: _matchPrefix, + // ignore: avoid_redundant_argument_values dont break things + matchSuffix: _matchSuffix, + ); + }); + + if (articlesAsync.hasValue && (articlesAsync.valueOrNull.isEmpty)) { + return const SliverToBoxAdapter(child: SizedBox.shrink()); + } + + return MultiSliver( + children: [ + SliverToBoxAdapter( + child: Padding( + padding: const EdgeInsets.only(left: 16.0), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text('Articles', style: Theme.of(context).textTheme.labelSmall), + ], + ), + ), + ), + SliverSkeletonizer( + enabled: articlesAsync.isLoading, + child: articlesAsync.when( + data: + (articles) => SliverList.builder( + itemCount: articles.length, + itemBuilder: (context, index) { + final article = articles[index]; + + final titleHighlight = switch (article) { + final FeedArticleQueryResult result => + result.titleHighlight.whenNotEmpty, + _ => null, + }; + + final searchSnippet = switch (article) { + final FeedArticleQueryResult result => + result.summarySnippet.whenNotEmpty ?? + result.contentSnippet.whenNotEmpty, + _ => null, + }; + + return ListTile( + leading: RepaintBoundary( + child: UrlIcon([ + article.icon ?? + article.links + ?.getRelation(FeedLinkRelation.alternate) + ?.uri ?? + article.siteLink ?? + article.feedId.base, + ], iconSize: 24.0), + ), + title: + (titleHighlight.isNotEmpty) + ? MarkdownBody( + data: titleHighlight!, + styleSheet: MarkdownStyleSheet( + p: Theme.of( + context, + ).textTheme.titleMedium?.copyWith( + color: + Theme.of(context).colorScheme.onSurface, + ), + ), + ) + : Text( + article.displayTitle, + style: theme.textTheme.titleMedium, + ), + subtitle: + (searchSnippet.isNotEmpty) + ? MarkdownBody( + data: searchSnippet!, + styleSheet: MarkdownStyleSheet( + p: Theme.of( + context, + ).textTheme.bodyMedium?.copyWith( + color: + Theme.of( + context, + ).colorScheme.onSurfaceVariant, + ), + a: Theme.of( + context, + ).textTheme.bodyMedium?.copyWith( + color: + Theme.of( + context, + ).colorScheme.onSurfaceVariant, + decoration: TextDecoration.none, + ), + ), + ) + : ((article.summaryPlain != null) + ? Text( + article.summaryPlain!, + style: theme.textTheme.bodySmall, + maxLines: 3, + overflow: TextOverflow.ellipsis, + ) + : null), + onTap: () { + FeedArticleRoute( + articleId: article.id, + ).pushReplacement(context); + }, + ); + }, + ), + error: (error, stackTrace) { + return SliverToBoxAdapter( + child: FailureWidget( + title: 'Failed searching Articles', + exception: error, + ), + ); + }, + loading: + () => SliverList.builder( + itemCount: articlesAsync.valueOrNull?.length ?? 3, + itemBuilder: (context, index) { + return const ListTile(title: Bone.text()); + }, + ), + ), + ), + ], + ); + } +} diff --git a/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart b/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart index aa165bf7..62d6caee 100644 --- a/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart +++ b/app/lib/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart @@ -20,7 +20,7 @@ class HistorySuggestions extends HookConsumerWidget { @override Widget build(BuildContext context, WidgetRef ref) { - final historySuggestions = ref.watch(engineHistorySuggestionsProvider); + final historySuggestionsAsync = ref.watch(engineHistorySuggestionsProvider); useListenableCallback(searchTextController, () async { await ref @@ -28,8 +28,8 @@ class HistorySuggestions extends HookConsumerWidget { .addQuery(searchTextController.text); }); - if (historySuggestions.hasValue && - (historySuggestions.valueOrNull.isEmpty)) { + if (historySuggestionsAsync.hasValue && + (historySuggestionsAsync.valueOrNull.isEmpty)) { return const SliverToBoxAdapter(child: SizedBox.shrink()); } @@ -46,8 +46,8 @@ class HistorySuggestions extends HookConsumerWidget { ), ), SliverSkeletonizer( - enabled: historySuggestions.isLoading, - child: historySuggestions.when( + enabled: historySuggestionsAsync.isLoading, + child: historySuggestionsAsync.when( skipLoadingOnReload: true, data: (historySuggestions) { return SliverList.builder( @@ -109,14 +109,16 @@ class HistorySuggestions extends HookConsumerWidget { ); }, error: (error, stackTrace) { - return FailureWidget( - title: 'Could not load history', - exception: error, + return SliverToBoxAdapter( + child: FailureWidget( + title: 'Could not load history', + exception: error, + ), ); }, loading: () => SliverList.builder( - itemCount: historySuggestions.valueOrNull?.length ?? 3, + itemCount: historySuggestionsAsync.valueOrNull?.length ?? 3, itemBuilder: (context, index) { return const ListTile(title: Bone.text()); }, diff --git a/app/lib/features/geckoview/features/tabs/data/providers.dart b/app/lib/features/geckoview/features/tabs/data/providers.dart index 217bc32c..ff313df1 100644 --- a/app/lib/features/geckoview/features/tabs/data/providers.dart +++ b/app/lib/features/geckoview/features/tabs/data/providers.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:drift/drift.dart'; import 'package:drift/native.dart'; import 'package:lensai/core/providers/models.dart'; @@ -51,8 +49,8 @@ TabDatabase tabDatabase(Ref ref) { embeddingDimensions: dimensions, ); - ref.onDispose(() { - unawaited(db.close()); + ref.onDispose(() async { + await db.close(); }); return db; diff --git a/app/lib/features/geckoview/features/tabs/data/providers.g.dart b/app/lib/features/geckoview/features/tabs/data/providers.g.dart index 1e3c0309..701bb61a 100644 --- a/app/lib/features/geckoview/features/tabs/data/providers.g.dart +++ b/app/lib/features/geckoview/features/tabs/data/providers.g.dart @@ -6,7 +6,7 @@ part of 'providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$tabDatabaseHash() => r'940dd2a1f1df6f2a0b77a1a3e9b8318550b832d2'; +String _$tabDatabaseHash() => r'db190f01d7e70e65b9f44403effa82ec503144a0'; /// See also [tabDatabase]. @ProviderFor(tabDatabase) diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart index 84ed2fd5..b2ae74c9 100644 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart +++ b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart @@ -108,8 +108,8 @@ class DocumentRepository extends _$DocumentRepository { } }); - ref.onDispose(() { - unawaited(tabContentSub.cancel()); + ref.onDispose(() async { + await tabContentSub.cancel(); }); } } diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart index 1a081ad3..b46e0fa0 100644 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart +++ b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart @@ -7,7 +7,7 @@ part of 'document.dart'; // ************************************************************************** String _$documentRepositoryHash() => - r'60e5a0c889d9659c8e0e254c59225c2041a0cb56'; + r'fd78b5eaf96951724e905b068144dcf2a254dcbb'; /// See also [DocumentRepository]. @ProviderFor(DocumentRepository) diff --git a/app/lib/features/tor/domain/services/tor_proxy.dart b/app/lib/features/tor/domain/services/tor_proxy.dart index a4dc9aa8..55b4609f 100644 --- a/app/lib/features/tor/domain/services/tor_proxy.dart +++ b/app/lib/features/tor/domain/services/tor_proxy.dart @@ -71,8 +71,8 @@ class TorProxyService extends _$TorProxyService { await _tor.initializeService(); - ref.onDispose(() { - unawaited(portSub.cancel()); + ref.onDispose(() async { + await portSub.cancel(); _tor.dispose(); }); diff --git a/app/lib/features/tor/domain/services/tor_proxy.g.dart b/app/lib/features/tor/domain/services/tor_proxy.g.dart index bf2f74c8..98256ba5 100644 --- a/app/lib/features/tor/domain/services/tor_proxy.g.dart +++ b/app/lib/features/tor/domain/services/tor_proxy.g.dart @@ -6,7 +6,7 @@ part of 'tor_proxy.dart'; // RiverpodGenerator // ************************************************************************** -String _$torProxyServiceHash() => r'85ed8e3b49007aa56e6b82ff802a3e0ce567a0c8'; +String _$torProxyServiceHash() => r'782708b8e5159931cecf8d32b843cb96a929fb1f'; /// See also [TorProxyService]. @ProviderFor(TorProxyService) diff --git a/app/lib/features/user/data/providers.dart b/app/lib/features/user/data/providers.dart index f8cfffbe..8545f283 100644 --- a/app/lib/features/user/data/providers.dart +++ b/app/lib/features/user/data/providers.dart @@ -1,5 +1,3 @@ -import 'dart:async'; - import 'package:drift/drift.dart'; import 'package:drift/native.dart'; import 'package:lensai/features/user/data/database/database.dart'; @@ -38,8 +36,8 @@ UserDatabase userDatabase(Ref ref) { }), ); - ref.onDispose(() { - unawaited(db.close()); + ref.onDispose(() async { + await db.close(); }); return db; diff --git a/app/lib/features/user/data/providers.g.dart b/app/lib/features/user/data/providers.g.dart index a3c95d2a..38535248 100644 --- a/app/lib/features/user/data/providers.g.dart +++ b/app/lib/features/user/data/providers.g.dart @@ -6,7 +6,7 @@ part of 'providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$userDatabaseHash() => r'717de293750f5da91dc7da3f964bd07f5dd61e10'; +String _$userDatabaseHash() => r'b925780435806f0241d7ddb635e301e9fc8baf6e'; /// See also [userDatabase]. @ProviderFor(userDatabase) diff --git a/app/lib/features/user/domain/repositories/engine_settings.dart b/app/lib/features/user/domain/repositories/engine_settings.dart index fbcee2ee..c1f1b25e 100644 --- a/app/lib/features/user/domain/repositories/engine_settings.dart +++ b/app/lib/features/user/domain/repositories/engine_settings.dart @@ -85,8 +85,8 @@ class EngineSettingsRepository extends _$EngineSettingsRepository { }); }); - ref.onDispose(() { - unawaited(watchSub.cancel()); + ref.onDispose(() async { + await watchSub.cancel(); }); return EngineSettings.withDefaults(); diff --git a/app/lib/features/user/domain/repositories/engine_settings.g.dart b/app/lib/features/user/domain/repositories/engine_settings.g.dart index dea7e338..23779dfb 100644 --- a/app/lib/features/user/domain/repositories/engine_settings.g.dart +++ b/app/lib/features/user/domain/repositories/engine_settings.g.dart @@ -7,7 +7,7 @@ part of 'engine_settings.dart'; // ************************************************************************** String _$engineSettingsRepositoryHash() => - r'940680d6d59a4a4a496589ac02eeab5a22779e5e'; + r'bd255f91f7cee702d9cfcb10bfdf3cd013234901'; /// See also [EngineSettingsRepository]. @ProviderFor(EngineSettingsRepository) diff --git a/app/lib/features/user/domain/repositories/general_settings.dart b/app/lib/features/user/domain/repositories/general_settings.dart index ab01f2b3..6b3a6d4e 100644 --- a/app/lib/features/user/domain/repositories/general_settings.dart +++ b/app/lib/features/user/domain/repositories/general_settings.dart @@ -74,8 +74,8 @@ class GeneralSettingsRepository extends _$GeneralSettingsRepository { state = _deserializeSettings(event); }); - ref.onDispose(() { - unawaited(watchSub.cancel()); + ref.onDispose(() async { + await watchSub.cancel(); }); return GeneralSettings.withDefaults(); diff --git a/app/lib/features/user/domain/repositories/general_settings.g.dart b/app/lib/features/user/domain/repositories/general_settings.g.dart index 91bc6261..92650e4c 100644 --- a/app/lib/features/user/domain/repositories/general_settings.g.dart +++ b/app/lib/features/user/domain/repositories/general_settings.g.dart @@ -7,7 +7,7 @@ part of 'general_settings.dart'; // ************************************************************************** String _$generalSettingsRepositoryHash() => - r'fd40dc270c52b137a5459f93f93c60e4c1ba9b58'; + r'001ffa37f5afe1d2d3ee2b3ce8539836af76cf89'; /// See also [GeneralSettingsRepository]. @ProviderFor(GeneralSettingsRepository) diff --git a/app/lib/features/web_feed/data/database/daos/article.dart b/app/lib/features/web_feed/data/database/daos/article.dart index 143d16e4..9f4f47c1 100644 --- a/app/lib/features/web_feed/data/database/daos/article.dart +++ b/app/lib/features/web_feed/data/database/daos/article.dart @@ -24,10 +24,45 @@ class ArticleDao extends DatabaseAccessor with _$ArticleDaoMixin { ]); } + Selectable getUnprocessedArticles() { + return db.articleView.select()..where( + (article) => + (article.contentHtml.isNotNull() & + (article.contentMarkdown.isNull() | + article.contentPlain.isNull())) | + (article.summaryHtml.isNotNull() & + (article.summaryMarkdown.isNull() | + article.summaryPlain.isNull())), + ); + } + SingleOrNullSelectable getArticleById(String articleId) { return db.articleView.select()..where((row) => row.id.equals(articleId)); } + Future updateArticleContent(List articles) { + return db.transaction(() async { + await Future.wait( + articles.map((newArticle) { + final statement = + db.article.update() + ..where((article) => article.id.equals(newArticle.id)); + + return statement.write( + ArticleCompanion( + summaryHtml: Value(newArticle.summaryHtml), + summaryMarkdown: Value(newArticle.summaryMarkdown), + summaryPlain: Value(newArticle.summaryPlain), + contentHtml: Value(newArticle.contentHtml), + contentMarkdown: Value(newArticle.contentMarkdown), + contentPlain: Value(newArticle.contentPlain), + ), + ); + }), + ); + }); + } + Future upsertArticles(List articles) { return db.transaction(() async { await Future.wait( @@ -39,9 +74,11 @@ class ArticleDao extends DatabaseAccessor with _$ArticleDaoMixin { (old) { return ArticleCompanion( authors: Value(article.authors), + contentHtml: Value(article.contentHtml), contentMarkdown: Value(article.contentMarkdown), contentPlain: Value(article.contentPlain), links: Value(article.links), + summaryHtml: Value(article.summaryHtml), summaryMarkdown: Value(article.summaryMarkdown), summaryPlain: Value(article.summaryPlain), tags: Value(article.tags), diff --git a/app/lib/features/web_feed/data/database/database.drift b/app/lib/features/web_feed/data/database/database.drift index f87edb49..0ab81519 100644 --- a/app/lib/features/web_feed/data/database/database.drift +++ b/app/lib/features/web_feed/data/database/database.drift @@ -27,8 +27,10 @@ CREATE TABLE article ( authors TEXT MAPPED BY `const FeedAuthorsConverter()`, tags TEXT MAPPED BY `const FeedCategoriesConverter()`, links TEXT MAPPED BY `const FeedLinksConverter()`, + summaryHtml TEXT, summaryMarkdown TEXT, summaryPlain TEXT, + contentHtml TEXT, contentMarkdown TEXT, contentPlain TEXT ) WITH FeedArticle; @@ -36,7 +38,8 @@ CREATE TABLE article ( CREATE VIEW article_view WITH FeedArticle AS SELECT a.*, - f.icon + f.icon, + f.site_link FROM article a INNER JOIN diff --git a/app/lib/features/web_feed/data/database/database.g.dart b/app/lib/features/web_feed/data/database/database.g.dart index 9fa10ad7..f3db572e 100644 --- a/app/lib/features/web_feed/data/database/database.g.dart +++ b/app/lib/features/web_feed/data/database/database.g.dart @@ -547,6 +547,14 @@ class Article extends Table with TableInfo { requiredDuringInsert: false, $customConstraints: '', ).withConverter?>(Article.$converterlinksn); + late final GeneratedColumn summaryHtml = GeneratedColumn( + 'summaryHtml', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + $customConstraints: '', + ); late final GeneratedColumn summaryMarkdown = GeneratedColumn( 'summaryMarkdown', aliasedName, @@ -563,6 +571,14 @@ class Article extends Table with TableInfo { requiredDuringInsert: false, $customConstraints: '', ); + late final GeneratedColumn contentHtml = GeneratedColumn( + 'contentHtml', + aliasedName, + true, + type: DriftSqlType.string, + requiredDuringInsert: false, + $customConstraints: '', + ); late final GeneratedColumn contentMarkdown = GeneratedColumn( 'contentMarkdown', aliasedName, @@ -591,8 +607,10 @@ class Article extends Table with TableInfo { authors, tags, links, + summaryHtml, summaryMarkdown, summaryPlain, + contentHtml, contentMarkdown, contentPlain, ]; @@ -657,6 +675,10 @@ class Article extends Table with TableInfo { data['${effectivePrefix}links'], ), ), + summaryHtml: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}summaryHtml'], + ), summaryMarkdown: attachedDatabase.typeMapping.read( DriftSqlType.string, data['${effectivePrefix}summaryMarkdown'], @@ -665,6 +687,10 @@ class Article extends Table with TableInfo { DriftSqlType.string, data['${effectivePrefix}summaryPlain'], ), + contentHtml: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}contentHtml'], + ), contentMarkdown: attachedDatabase.typeMapping.read( DriftSqlType.string, data['${effectivePrefix}contentMarkdown'], @@ -709,8 +735,10 @@ class ArticleCompanion extends UpdateCompanion { final Value?> authors; final Value?> tags; final Value?> links; + final Value summaryHtml; final Value summaryMarkdown; final Value summaryPlain; + final Value contentHtml; final Value contentMarkdown; final Value contentPlain; final Value rowid; @@ -725,8 +753,10 @@ class ArticleCompanion extends UpdateCompanion { this.authors = const Value.absent(), this.tags = const Value.absent(), this.links = const Value.absent(), + this.summaryHtml = const Value.absent(), this.summaryMarkdown = const Value.absent(), this.summaryPlain = const Value.absent(), + this.contentHtml = const Value.absent(), this.contentMarkdown = const Value.absent(), this.contentPlain = const Value.absent(), this.rowid = const Value.absent(), @@ -742,8 +772,10 @@ class ArticleCompanion extends UpdateCompanion { this.authors = const Value.absent(), this.tags = const Value.absent(), this.links = const Value.absent(), + this.summaryHtml = const Value.absent(), this.summaryMarkdown = const Value.absent(), this.summaryPlain = const Value.absent(), + this.contentHtml = const Value.absent(), this.contentMarkdown = const Value.absent(), this.contentPlain = const Value.absent(), this.rowid = const Value.absent(), @@ -761,8 +793,10 @@ class ArticleCompanion extends UpdateCompanion { Expression? authors, Expression? tags, Expression? links, + Expression? summaryHtml, Expression? summaryMarkdown, Expression? summaryPlain, + Expression? contentHtml, Expression? contentMarkdown, Expression? contentPlain, Expression? rowid, @@ -778,8 +812,10 @@ class ArticleCompanion extends UpdateCompanion { if (authors != null) 'authors': authors, if (tags != null) 'tags': tags, if (links != null) 'links': links, + if (summaryHtml != null) 'summaryHtml': summaryHtml, if (summaryMarkdown != null) 'summaryMarkdown': summaryMarkdown, if (summaryPlain != null) 'summaryPlain': summaryPlain, + if (contentHtml != null) 'contentHtml': contentHtml, if (contentMarkdown != null) 'contentMarkdown': contentMarkdown, if (contentPlain != null) 'contentPlain': contentPlain, if (rowid != null) 'rowid': rowid, @@ -797,8 +833,10 @@ class ArticleCompanion extends UpdateCompanion { Value?>? authors, Value?>? tags, Value?>? links, + Value? summaryHtml, Value? summaryMarkdown, Value? summaryPlain, + Value? contentHtml, Value? contentMarkdown, Value? contentPlain, Value? rowid, @@ -814,8 +852,10 @@ class ArticleCompanion extends UpdateCompanion { authors: authors ?? this.authors, tags: tags ?? this.tags, links: links ?? this.links, + summaryHtml: summaryHtml ?? this.summaryHtml, summaryMarkdown: summaryMarkdown ?? this.summaryMarkdown, summaryPlain: summaryPlain ?? this.summaryPlain, + contentHtml: contentHtml ?? this.contentHtml, contentMarkdown: contentMarkdown ?? this.contentMarkdown, contentPlain: contentPlain ?? this.contentPlain, rowid: rowid ?? this.rowid, @@ -861,12 +901,18 @@ class ArticleCompanion extends UpdateCompanion { Article.$converterlinksn.toSql(links.value), ); } + if (summaryHtml.present) { + map['summaryHtml'] = Variable(summaryHtml.value); + } if (summaryMarkdown.present) { map['summaryMarkdown'] = Variable(summaryMarkdown.value); } if (summaryPlain.present) { map['summaryPlain'] = Variable(summaryPlain.value); } + if (contentHtml.present) { + map['contentHtml'] = Variable(contentHtml.value); + } if (contentMarkdown.present) { map['contentMarkdown'] = Variable(contentMarkdown.value); } @@ -892,8 +938,10 @@ class ArticleCompanion extends UpdateCompanion { ..write('authors: $authors, ') ..write('tags: $tags, ') ..write('links: $links, ') + ..write('summaryHtml: $summaryHtml, ') ..write('summaryMarkdown: $summaryMarkdown, ') ..write('summaryPlain: $summaryPlain, ') + ..write('contentHtml: $contentHtml, ') ..write('contentMarkdown: $contentMarkdown, ') ..write('contentPlain: $contentPlain, ') ..write('rowid: $rowid') @@ -920,8 +968,10 @@ class ArticleView extends ViewInfo authors, tags, links, + summaryHtml, summaryMarkdown, summaryPlain, + contentHtml, contentMarkdown, contentPlain, icon, @@ -991,6 +1041,10 @@ class ArticleView extends ViewInfo data['${effectivePrefix}links'], ), ), + summaryHtml: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}summaryHtml'], + ), summaryMarkdown: attachedDatabase.typeMapping.read( DriftSqlType.string, data['${effectivePrefix}summaryMarkdown'], @@ -999,6 +1053,10 @@ class ArticleView extends ViewInfo DriftSqlType.string, data['${effectivePrefix}summaryPlain'], ), + contentHtml: attachedDatabase.typeMapping.read( + DriftSqlType.string, + data['${effectivePrefix}contentHtml'], + ), contentMarkdown: attachedDatabase.typeMapping.read( DriftSqlType.string, data['${effectivePrefix}contentMarkdown'], @@ -1080,6 +1138,12 @@ class ArticleView extends ViewInfo true, type: DriftSqlType.string, ).withConverter?>(Article.$converterlinksn); + late final GeneratedColumn summaryHtml = GeneratedColumn( + 'summaryHtml', + aliasedName, + true, + type: DriftSqlType.string, + ); late final GeneratedColumn summaryMarkdown = GeneratedColumn( 'summaryMarkdown', aliasedName, @@ -1092,6 +1156,12 @@ class ArticleView extends ViewInfo true, type: DriftSqlType.string, ); + late final GeneratedColumn contentHtml = GeneratedColumn( + 'contentHtml', + aliasedName, + true, + type: DriftSqlType.string, + ); late final GeneratedColumn contentMarkdown = GeneratedColumn( 'contentMarkdown', aliasedName, @@ -1420,8 +1490,10 @@ abstract class _$FeedDatabase extends GeneratedDatabase { Article.$converterlinks, row.readNullable('links'), ), + summaryHtml: row.readNullable('summaryHtml'), summaryMarkdown: row.readNullable('summaryMarkdown'), summaryPlain: row.readNullable('summaryPlain'), + contentHtml: row.readNullable('contentHtml'), contentMarkdown: row.readNullable('contentMarkdown'), contentPlain: row.readNullable('contentPlain'), icon: NullAwareTypeConverter.wrapFromSql( @@ -1473,8 +1545,10 @@ abstract class _$FeedDatabase extends GeneratedDatabase { Article.$converterlinks, row.readNullable('links'), ), + summaryHtml: row.readNullable('summaryHtml'), summaryMarkdown: row.readNullable('summaryMarkdown'), summaryPlain: row.readNullable('summaryPlain'), + contentHtml: row.readNullable('contentHtml'), contentMarkdown: row.readNullable('contentMarkdown'), contentPlain: row.readNullable('contentPlain'), icon: NullAwareTypeConverter.wrapFromSql( @@ -1911,8 +1985,10 @@ typedef $ArticleCreateCompanionBuilder = Value?> authors, Value?> tags, Value?> links, + Value summaryHtml, Value summaryMarkdown, Value summaryPlain, + Value contentHtml, Value contentMarkdown, Value contentPlain, Value rowid, @@ -1929,8 +2005,10 @@ typedef $ArticleUpdateCompanionBuilder = Value?> authors, Value?> tags, Value?> links, + Value summaryHtml, Value summaryMarkdown, Value summaryPlain, + Value contentHtml, Value contentMarkdown, Value contentPlain, Value rowid, @@ -2018,6 +2096,11 @@ class $ArticleFilterComposer extends Composer<_$FeedDatabase, Article> { builder: (column) => ColumnWithTypeConverterFilters(column), ); + ColumnFilters get summaryHtml => $composableBuilder( + column: $table.summaryHtml, + builder: (column) => ColumnFilters(column), + ); + ColumnFilters get summaryMarkdown => $composableBuilder( column: $table.summaryMarkdown, builder: (column) => ColumnFilters(column), @@ -2028,6 +2111,11 @@ class $ArticleFilterComposer extends Composer<_$FeedDatabase, Article> { builder: (column) => ColumnFilters(column), ); + ColumnFilters get contentHtml => $composableBuilder( + column: $table.contentHtml, + builder: (column) => ColumnFilters(column), + ); + ColumnFilters get contentMarkdown => $composableBuilder( column: $table.contentMarkdown, builder: (column) => ColumnFilters(column), @@ -2115,6 +2203,11 @@ class $ArticleOrderingComposer extends Composer<_$FeedDatabase, Article> { builder: (column) => ColumnOrderings(column), ); + ColumnOrderings get summaryHtml => $composableBuilder( + column: $table.summaryHtml, + builder: (column) => ColumnOrderings(column), + ); + ColumnOrderings get summaryMarkdown => $composableBuilder( column: $table.summaryMarkdown, builder: (column) => ColumnOrderings(column), @@ -2125,6 +2218,11 @@ class $ArticleOrderingComposer extends Composer<_$FeedDatabase, Article> { builder: (column) => ColumnOrderings(column), ); + ColumnOrderings get contentHtml => $composableBuilder( + column: $table.contentHtml, + builder: (column) => ColumnOrderings(column), + ); + ColumnOrderings get contentMarkdown => $composableBuilder( column: $table.contentMarkdown, builder: (column) => ColumnOrderings(column), @@ -2194,6 +2292,11 @@ class $ArticleAnnotationComposer extends Composer<_$FeedDatabase, Article> { GeneratedColumnWithTypeConverter?, String> get links => $composableBuilder(column: $table.links, builder: (column) => column); + GeneratedColumn get summaryHtml => $composableBuilder( + column: $table.summaryHtml, + builder: (column) => column, + ); + GeneratedColumn get summaryMarkdown => $composableBuilder( column: $table.summaryMarkdown, builder: (column) => column, @@ -2204,6 +2307,11 @@ class $ArticleAnnotationComposer extends Composer<_$FeedDatabase, Article> { builder: (column) => column, ); + GeneratedColumn get contentHtml => $composableBuilder( + column: $table.contentHtml, + builder: (column) => column, + ); + GeneratedColumn get contentMarkdown => $composableBuilder( column: $table.contentMarkdown, builder: (column) => column, @@ -2276,8 +2384,10 @@ class $ArticleTableManager Value?> authors = const Value.absent(), Value?> tags = const Value.absent(), Value?> links = const Value.absent(), + Value summaryHtml = const Value.absent(), Value summaryMarkdown = const Value.absent(), Value summaryPlain = const Value.absent(), + Value contentHtml = const Value.absent(), Value contentMarkdown = const Value.absent(), Value contentPlain = const Value.absent(), Value rowid = const Value.absent(), @@ -2292,8 +2402,10 @@ class $ArticleTableManager authors: authors, tags: tags, links: links, + summaryHtml: summaryHtml, summaryMarkdown: summaryMarkdown, summaryPlain: summaryPlain, + contentHtml: contentHtml, contentMarkdown: contentMarkdown, contentPlain: contentPlain, rowid: rowid, @@ -2310,8 +2422,10 @@ class $ArticleTableManager Value?> authors = const Value.absent(), Value?> tags = const Value.absent(), Value?> links = const Value.absent(), + Value summaryHtml = const Value.absent(), Value summaryMarkdown = const Value.absent(), Value summaryPlain = const Value.absent(), + Value contentHtml = const Value.absent(), Value contentMarkdown = const Value.absent(), Value contentPlain = const Value.absent(), Value rowid = const Value.absent(), @@ -2326,8 +2440,10 @@ class $ArticleTableManager authors: authors, tags: tags, links: links, + summaryHtml: summaryHtml, summaryMarkdown: summaryMarkdown, summaryPlain: summaryPlain, + contentHtml: contentHtml, contentMarkdown: contentMarkdown, contentPlain: contentPlain, rowid: rowid, diff --git a/app/lib/features/web_feed/data/models/feed_article.dart b/app/lib/features/web_feed/data/models/feed_article.dart index 2d48fc9c..ac3459b7 100644 --- a/app/lib/features/web_feed/data/models/feed_article.dart +++ b/app/lib/features/web_feed/data/models/feed_article.dart @@ -1,3 +1,4 @@ +import 'package:copy_with_extension/copy_with_extension.dart'; import 'package:drift/drift.dart'; import 'package:fast_equatable/fast_equatable.dart'; import 'package:json_annotation/json_annotation.dart'; @@ -9,6 +10,7 @@ import 'package:lensai/features/web_feed/data/models/feed_link.dart'; part 'feed_article.g.dart'; @JsonSerializable() +@CopyWith() class FeedArticle with FastEquatable implements Insertable { final String id; final Uri feedId; @@ -20,13 +22,16 @@ class FeedArticle with FastEquatable implements Insertable { final List? authors; final List? tags; final List? links; + final String? summaryHtml; final String? summaryMarkdown; final String? summaryPlain; + final String? contentHtml; final String? contentMarkdown; final String? contentPlain; //Derived by view from feed table, should not get inserted final Uri? icon; + final Uri? siteLink; FeedArticle({ required this.id, @@ -39,11 +44,14 @@ class FeedArticle with FastEquatable implements Insertable { this.authors, this.tags, this.links, + this.summaryHtml, this.summaryMarkdown, this.summaryPlain, + this.contentHtml, this.contentMarkdown, this.contentPlain, this.icon, + this.siteLink, }); factory FeedArticle.fromJson(Map json) => @@ -82,12 +90,18 @@ class FeedArticle with FastEquatable implements Insertable { if (!nullToAbsent || links != null) { map['links'] = Variable(Article.$converterlinksn.toSql(links)); } + if (!nullToAbsent || summaryHtml != null) { + map['summaryHtml'] = Variable(summaryHtml); + } if (!nullToAbsent || summaryMarkdown != null) { map['summaryMarkdown'] = Variable(summaryMarkdown); } if (!nullToAbsent || summaryPlain != null) { map['summaryPlain'] = Variable(summaryPlain); } + if (!nullToAbsent || contentHtml != null) { + map['contentHtml'] = Variable(contentHtml); + } if (!nullToAbsent || contentMarkdown != null) { map['contentMarkdown'] = Variable(contentMarkdown); } @@ -109,10 +123,13 @@ class FeedArticle with FastEquatable implements Insertable { authors, tags, links, + summaryHtml, summaryMarkdown, summaryPlain, + contentHtml, contentMarkdown, contentPlain, icon, + siteLink, ]; } diff --git a/app/lib/features/web_feed/data/models/feed_article.g.dart b/app/lib/features/web_feed/data/models/feed_article.g.dart index d12f8a72..ffe24b6e 100644 --- a/app/lib/features/web_feed/data/models/feed_article.g.dart +++ b/app/lib/features/web_feed/data/models/feed_article.g.dart @@ -2,6 +2,257 @@ part of 'feed_article.dart'; +// ************************************************************************** +// CopyWithGenerator +// ************************************************************************** + +abstract class _$FeedArticleCWProxy { + FeedArticle id(String id); + + FeedArticle feedId(Uri feedId); + + FeedArticle fetched(DateTime fetched); + + FeedArticle created(DateTime? created); + + FeedArticle updated(DateTime? updated); + + FeedArticle lastRead(DateTime? lastRead); + + FeedArticle title(String? title); + + FeedArticle authors(List? authors); + + FeedArticle tags(List? tags); + + FeedArticle links(List? links); + + FeedArticle summaryHtml(String? summaryHtml); + + FeedArticle summaryMarkdown(String? summaryMarkdown); + + FeedArticle summaryPlain(String? summaryPlain); + + FeedArticle contentHtml(String? contentHtml); + + FeedArticle contentMarkdown(String? contentMarkdown); + + FeedArticle contentPlain(String? contentPlain); + + FeedArticle icon(Uri? icon); + + /// This function **does support** nullification of nullable fields. All `null` values passed to `non-nullable` fields will be ignored. You can also use `FeedArticle(...).copyWith.fieldName(...)` to override fields one at a time with nullification support. + /// + /// Usage + /// ```dart + /// FeedArticle(...).copyWith(id: 12, name: "My name") + /// ```` + FeedArticle call({ + String id, + Uri feedId, + DateTime fetched, + DateTime? created, + DateTime? updated, + DateTime? lastRead, + String? title, + List? authors, + List? tags, + List? links, + String? summaryHtml, + String? summaryMarkdown, + String? summaryPlain, + String? contentHtml, + String? contentMarkdown, + String? contentPlain, + Uri? icon, + }); +} + +/// Proxy class for `copyWith` functionality. This is a callable class and can be used as follows: `instanceOfFeedArticle.copyWith(...)`. Additionally contains functions for specific fields e.g. `instanceOfFeedArticle.copyWith.fieldName(...)` +class _$FeedArticleCWProxyImpl implements _$FeedArticleCWProxy { + const _$FeedArticleCWProxyImpl(this._value); + + final FeedArticle _value; + + @override + FeedArticle id(String id) => this(id: id); + + @override + FeedArticle feedId(Uri feedId) => this(feedId: feedId); + + @override + FeedArticle fetched(DateTime fetched) => this(fetched: fetched); + + @override + FeedArticle created(DateTime? created) => this(created: created); + + @override + FeedArticle updated(DateTime? updated) => this(updated: updated); + + @override + FeedArticle lastRead(DateTime? lastRead) => this(lastRead: lastRead); + + @override + FeedArticle title(String? title) => this(title: title); + + @override + FeedArticle authors(List? authors) => this(authors: authors); + + @override + FeedArticle tags(List? tags) => this(tags: tags); + + @override + FeedArticle links(List? links) => this(links: links); + + @override + FeedArticle summaryHtml(String? summaryHtml) => + this(summaryHtml: summaryHtml); + + @override + FeedArticle summaryMarkdown(String? summaryMarkdown) => + this(summaryMarkdown: summaryMarkdown); + + @override + FeedArticle summaryPlain(String? summaryPlain) => + this(summaryPlain: summaryPlain); + + @override + FeedArticle contentHtml(String? contentHtml) => + this(contentHtml: contentHtml); + + @override + FeedArticle contentMarkdown(String? contentMarkdown) => + this(contentMarkdown: contentMarkdown); + + @override + FeedArticle contentPlain(String? contentPlain) => + this(contentPlain: contentPlain); + + @override + FeedArticle icon(Uri? icon) => this(icon: icon); + + @override + /// This function **does support** nullification of nullable fields. All `null` values passed to `non-nullable` fields will be ignored. You can also use `FeedArticle(...).copyWith.fieldName(...)` to override fields one at a time with nullification support. + /// + /// Usage + /// ```dart + /// FeedArticle(...).copyWith(id: 12, name: "My name") + /// ```` + FeedArticle call({ + Object? id = const $CopyWithPlaceholder(), + Object? feedId = const $CopyWithPlaceholder(), + Object? fetched = const $CopyWithPlaceholder(), + Object? created = const $CopyWithPlaceholder(), + Object? updated = const $CopyWithPlaceholder(), + Object? lastRead = const $CopyWithPlaceholder(), + Object? title = const $CopyWithPlaceholder(), + Object? authors = const $CopyWithPlaceholder(), + Object? tags = const $CopyWithPlaceholder(), + Object? links = const $CopyWithPlaceholder(), + Object? summaryHtml = const $CopyWithPlaceholder(), + Object? summaryMarkdown = const $CopyWithPlaceholder(), + Object? summaryPlain = const $CopyWithPlaceholder(), + Object? contentHtml = const $CopyWithPlaceholder(), + Object? contentMarkdown = const $CopyWithPlaceholder(), + Object? contentPlain = const $CopyWithPlaceholder(), + Object? icon = const $CopyWithPlaceholder(), + }) { + return FeedArticle( + id: + id == const $CopyWithPlaceholder() + ? _value.id + // ignore: cast_nullable_to_non_nullable + : id as String, + feedId: + feedId == const $CopyWithPlaceholder() + ? _value.feedId + // ignore: cast_nullable_to_non_nullable + : feedId as Uri, + fetched: + fetched == const $CopyWithPlaceholder() + ? _value.fetched + // ignore: cast_nullable_to_non_nullable + : fetched as DateTime, + created: + created == const $CopyWithPlaceholder() + ? _value.created + // ignore: cast_nullable_to_non_nullable + : created as DateTime?, + updated: + updated == const $CopyWithPlaceholder() + ? _value.updated + // ignore: cast_nullable_to_non_nullable + : updated as DateTime?, + lastRead: + lastRead == const $CopyWithPlaceholder() + ? _value.lastRead + // ignore: cast_nullable_to_non_nullable + : lastRead as DateTime?, + title: + title == const $CopyWithPlaceholder() + ? _value.title + // ignore: cast_nullable_to_non_nullable + : title as String?, + authors: + authors == const $CopyWithPlaceholder() + ? _value.authors + // ignore: cast_nullable_to_non_nullable + : authors as List?, + tags: + tags == const $CopyWithPlaceholder() + ? _value.tags + // ignore: cast_nullable_to_non_nullable + : tags as List?, + links: + links == const $CopyWithPlaceholder() + ? _value.links + // ignore: cast_nullable_to_non_nullable + : links as List?, + summaryHtml: + summaryHtml == const $CopyWithPlaceholder() + ? _value.summaryHtml + // ignore: cast_nullable_to_non_nullable + : summaryHtml as String?, + summaryMarkdown: + summaryMarkdown == const $CopyWithPlaceholder() + ? _value.summaryMarkdown + // ignore: cast_nullable_to_non_nullable + : summaryMarkdown as String?, + summaryPlain: + summaryPlain == const $CopyWithPlaceholder() + ? _value.summaryPlain + // ignore: cast_nullable_to_non_nullable + : summaryPlain as String?, + contentHtml: + contentHtml == const $CopyWithPlaceholder() + ? _value.contentHtml + // ignore: cast_nullable_to_non_nullable + : contentHtml as String?, + contentMarkdown: + contentMarkdown == const $CopyWithPlaceholder() + ? _value.contentMarkdown + // ignore: cast_nullable_to_non_nullable + : contentMarkdown as String?, + contentPlain: + contentPlain == const $CopyWithPlaceholder() + ? _value.contentPlain + // ignore: cast_nullable_to_non_nullable + : contentPlain as String?, + icon: + icon == const $CopyWithPlaceholder() + ? _value.icon + // ignore: cast_nullable_to_non_nullable + : icon as Uri?, + ); + } +} + +extension $FeedArticleCopyWith on FeedArticle { + /// Returns a callable class that can be used as follows: `instanceOfFeedArticle.copyWith(...)` or like so:`instanceOfFeedArticle.copyWith.fieldName(...)`. + // ignore: library_private_types_in_public_api + _$FeedArticleCWProxy get copyWith => _$FeedArticleCWProxyImpl(this); +} + // ************************************************************************** // JsonSerializableGenerator // ************************************************************************** @@ -35,8 +286,10 @@ FeedArticle _$FeedArticleFromJson(Map json) => FeedArticle( (json['links'] as List?) ?.map((e) => FeedLink.fromJson(e as Map)) .toList(), + summaryHtml: json['summaryHtml'] as String?, summaryMarkdown: json['summaryMarkdown'] as String?, summaryPlain: json['summaryPlain'] as String?, + contentHtml: json['contentHtml'] as String?, contentMarkdown: json['contentMarkdown'] as String?, contentPlain: json['contentPlain'] as String?, icon: json['icon'] == null ? null : Uri.parse(json['icon'] as String), @@ -54,8 +307,10 @@ Map _$FeedArticleToJson(FeedArticle instance) => 'authors': instance.authors?.map((e) => e.toJson()).toList(), 'tags': instance.tags?.map((e) => e.toJson()).toList(), 'links': instance.links?.map((e) => e.toJson()).toList(), + 'summaryHtml': instance.summaryHtml, 'summaryMarkdown': instance.summaryMarkdown, 'summaryPlain': instance.summaryPlain, + 'contentHtml': instance.contentHtml, 'contentMarkdown': instance.contentMarkdown, 'contentPlain': instance.contentPlain, 'icon': instance.icon?.toString(), diff --git a/app/lib/features/web_feed/data/models/feed_article_query_result.dart b/app/lib/features/web_feed/data/models/feed_article_query_result.dart index 29347dc5..4c33ca94 100644 --- a/app/lib/features/web_feed/data/models/feed_article_query_result.dart +++ b/app/lib/features/web_feed/data/models/feed_article_query_result.dart @@ -19,8 +19,10 @@ class FeedArticleQueryResult extends FeedArticle { required super.authors, required super.tags, required super.links, + required super.summaryHtml, required super.summaryMarkdown, required super.summaryPlain, + required super.contentHtml, required super.contentMarkdown, required super.contentPlain, required super.icon, diff --git a/app/lib/features/web_feed/data/providers.dart b/app/lib/features/web_feed/data/providers.dart index 3d6274c8..5a610500 100644 --- a/app/lib/features/web_feed/data/providers.dart +++ b/app/lib/features/web_feed/data/providers.dart @@ -1,4 +1,3 @@ -import 'dart:async'; import 'dart:io'; import 'package:drift/drift.dart'; @@ -38,8 +37,8 @@ FeedDatabase feedDatabase(Ref ref) { }), ); - ref.onDispose(() { - unawaited(db.close()); + ref.onDispose(() async { + await db.close(); }); return db; diff --git a/app/lib/features/web_feed/data/providers.g.dart b/app/lib/features/web_feed/data/providers.g.dart index 2afc7afa..38933295 100644 --- a/app/lib/features/web_feed/data/providers.g.dart +++ b/app/lib/features/web_feed/data/providers.g.dart @@ -6,7 +6,7 @@ part of 'providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$feedDatabaseHash() => r'a4a1ffa36b73c2aecbe17bf096e30fa9b9ad23c9'; +String _$feedDatabaseHash() => r'c3b20e867da5af6e92d1e8c1efa96b79988837a1'; /// See also [feedDatabase]. @ProviderFor(feedDatabase) diff --git a/app/lib/features/web_feed/domain/providers.dart b/app/lib/features/web_feed/domain/providers.dart index 75be22da..21ea960b 100644 --- a/app/lib/features/web_feed/domain/providers.dart +++ b/app/lib/features/web_feed/domain/providers.dart @@ -10,6 +10,7 @@ import 'package:lensai/features/web_feed/domain/repositories/feed_repository.dar import 'package:lensai/features/web_feed/domain/services/feed_reader.dart'; import 'package:riverpod/riverpod.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; +import 'package:rxdart/rxdart.dart'; part 'providers.g.dart'; @@ -53,7 +54,7 @@ class ArticleSearch extends _$ArticleSearch { await _streamController.close(); }); - return _streamController.stream; + return ConcatStream([Stream.value([]), _streamController.stream]); } } diff --git a/app/lib/features/web_feed/domain/providers.g.dart b/app/lib/features/web_feed/domain/providers.g.dart index 8f47cf92..0e942592 100644 --- a/app/lib/features/web_feed/domain/providers.g.dart +++ b/app/lib/features/web_feed/domain/providers.g.dart @@ -827,7 +827,7 @@ class _ArticleSearchProviderElement } String _$filteredArticleListHash() => - r'4443a4a92cab0c8f534a74a1cba681a8973b058e'; + r'a691c3c6aa722dd85ee4980e6c48721a8025f1d8'; abstract class _$FilteredArticleList extends BuildlessAutoDisposeNotifier>> { diff --git a/app/lib/features/web_feed/domain/services/article_content_processor.dart b/app/lib/features/web_feed/domain/services/article_content_processor.dart new file mode 100644 index 00000000..76e1ebe4 --- /dev/null +++ b/app/lib/features/web_feed/domain/services/article_content_processor.dart @@ -0,0 +1,51 @@ +import 'package:collection/collection.dart'; +import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; +import 'package:lensai/core/logger.dart'; +import 'package:lensai/extensions/nullable.dart'; +import 'package:lensai/features/web_feed/data/models/feed_article.dart'; +import 'package:lensai/features/web_feed/data/providers.dart'; +import 'package:riverpod_annotation/riverpod_annotation.dart'; + +part 'article_content_processor.g.dart'; + +@Riverpod(keepAlive: true) +class ArticleContentProcessorService extends _$ArticleContentProcessorService { + @override + void build() { + final db = ref.watch(feedDatabaseProvider); + + final processSub = db.articleDao.getUnprocessedArticles().watch().listen(( + articles, + ) async { + try { + final content = await GeckoBrowserExtensionService.turndownHtml( + articles.map((article) => article.contentHtml ?? '').toList(), + ); + final summary = await GeckoBrowserExtensionService.turndownHtml( + articles.map((article) => article.summaryHtml ?? '').toList(), + ); + + await db.articleDao.updateArticleContent( + articles + .mapIndexed( + (index, article) => article.copyWith( + contentMarkdown: content[index].markdown.whenNotEmpty, + contentPlain: content[index].plain.whenNotEmpty, + summaryMarkdown: summary[index].markdown.whenNotEmpty, + summaryPlain: summary[index].plain.whenNotEmpty, + ), + ) + .toList(), + ); + + logger.i('Processed ${articles.length} articles'); + } catch (e, s) { + logger.e('Error processing articles', error: e, stackTrace: s); + } + }); + + ref.onDispose(() async { + await processSub.cancel(); + }); + } +} diff --git a/app/lib/features/web_feed/domain/services/article_content_processor.g.dart b/app/lib/features/web_feed/domain/services/article_content_processor.g.dart new file mode 100644 index 00000000..391ad6df --- /dev/null +++ b/app/lib/features/web_feed/domain/services/article_content_processor.g.dart @@ -0,0 +1,28 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'article_content_processor.dart'; + +// ************************************************************************** +// RiverpodGenerator +// ************************************************************************** + +String _$articleContentProcessorServiceHash() => + r'182cd89e38a0a9a86b7425c16ad2bd1fc39c6704'; + +/// See also [ArticleContentProcessorService]. +@ProviderFor(ArticleContentProcessorService) +final articleContentProcessorServiceProvider = + NotifierProvider.internal( + ArticleContentProcessorService.new, + name: r'articleContentProcessorServiceProvider', + debugGetCreateSourceHash: + const bool.fromEnvironment('dart.vm.product') + ? null + : _$articleContentProcessorServiceHash, + dependencies: null, + allTransitiveDependencies: null, + ); + +typedef _$ArticleContentProcessorService = Notifier; +// ignore_for_file: type=lint +// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/web_feed/domain/services/feed_reader.dart b/app/lib/features/web_feed/domain/services/feed_reader.dart index fc5aa8df..21cdaeec 100644 --- a/app/lib/features/web_feed/domain/services/feed_reader.dart +++ b/app/lib/features/web_feed/domain/services/feed_reader.dart @@ -28,7 +28,7 @@ class FeedReader extends _$FeedReader { final parser = FeedParser.parse(url: url, xmlString: response.body); final result = FeedParseResult( feedData: parser.readGeneralData(), - articleData: await parser.readArticles(), + articleData: parser.readArticles(), ); return result.toJson(); diff --git a/app/lib/features/web_feed/domain/services/feed_reader.g.dart b/app/lib/features/web_feed/domain/services/feed_reader.g.dart index 0e66af89..1cddd500 100644 --- a/app/lib/features/web_feed/domain/services/feed_reader.g.dart +++ b/app/lib/features/web_feed/domain/services/feed_reader.g.dart @@ -6,7 +6,7 @@ part of 'feed_reader.dart'; // RiverpodGenerator // ************************************************************************** -String _$feedReaderHash() => r'67b09485e40c522108709ca32b9c624b4db2446b'; +String _$feedReaderHash() => r'301e01fa831c58723dc0ab118496f4fb34c38610'; /// See also [FeedReader]. @ProviderFor(FeedReader) diff --git a/app/lib/features/web_feed/presentation/controllers/fetch_articles.dart b/app/lib/features/web_feed/presentation/controllers/fetch_articles.dart index ca3ea0b3..92aaaa08 100644 --- a/app/lib/features/web_feed/presentation/controllers/fetch_articles.dart +++ b/app/lib/features/web_feed/presentation/controllers/fetch_articles.dart @@ -1,3 +1,4 @@ +import 'package:lensai/core/logger.dart'; import 'package:lensai/features/web_feed/domain/repositories/feed_repository.dart'; import 'package:lensai/features/web_feed/domain/services/feed_reader.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; @@ -14,17 +15,25 @@ class FetchArticlesController extends _$FetchArticlesController { await Future.wait( feeds.map((feed) async { - final result = await ref - .read(feedReaderProvider.notifier) - .parseFeed(feed.url); + try { + final result = await ref + .read(feedReaderProvider.notifier) + .parseFeed(feed.url); - await ref - .read(feedRepositoryProvider.notifier) - .upsertArticles(result.articleData); + await ref + .read(feedRepositoryProvider.notifier) + .upsertArticles(result.articleData); - await ref - .read(feedRepositoryProvider.notifier) - .touchFeedFetched(feed.url); + await ref + .read(feedRepositoryProvider.notifier) + .touchFeedFetched(feed.url); + } catch (e, s) { + logger.e( + 'Failed fetching feed ${feed.url}', + error: e, + stackTrace: s, + ); + } }).toList(), ); }); diff --git a/app/lib/features/web_feed/presentation/controllers/fetch_articles.g.dart b/app/lib/features/web_feed/presentation/controllers/fetch_articles.g.dart index aa9000ae..8458e999 100644 --- a/app/lib/features/web_feed/presentation/controllers/fetch_articles.g.dart +++ b/app/lib/features/web_feed/presentation/controllers/fetch_articles.g.dart @@ -7,7 +7,7 @@ part of 'fetch_articles.dart'; // ************************************************************************** String _$fetchArticlesControllerHash() => - r'63da7f6ba9eab28aebff4c60d98e2f2a404c5138'; + r'7604d9352da886624c3cf6862b89c397edda1f28'; /// See also [FetchArticlesController]. @ProviderFor(FetchArticlesController) diff --git a/app/lib/features/web_feed/presentation/widgets/feed_article_card.dart b/app/lib/features/web_feed/presentation/widgets/feed_article_card.dart index fe5a331a..2b715b11 100644 --- a/app/lib/features/web_feed/presentation/widgets/feed_article_card.dart +++ b/app/lib/features/web_feed/presentation/widgets/feed_article_card.dart @@ -6,8 +6,10 @@ import 'package:lensai/extensions/nullable.dart'; import 'package:lensai/extensions/uri.dart'; import 'package:lensai/features/web_feed/data/models/feed_article.dart'; import 'package:lensai/features/web_feed/data/models/feed_article_query_result.dart'; +import 'package:lensai/features/web_feed/data/models/feed_link.dart'; import 'package:lensai/features/web_feed/domain/providers/article_filter.dart'; import 'package:lensai/features/web_feed/domain/repositories/feed_repository.dart'; +import 'package:lensai/features/web_feed/extensions/atom.dart'; import 'package:lensai/features/web_feed/extensions/feed_article.dart'; import 'package:lensai/features/web_feed/presentation/widgets/authors_horizontal_list.dart'; import 'package:lensai/features/web_feed/presentation/widgets/tags_horizontal_list.dart'; @@ -52,7 +54,12 @@ class FeedArticleCard extends HookConsumerWidget { Row( children: [ UrlIcon([ - article.icon ?? article.feedId.base, + article.icon ?? + article.links + ?.getRelation(FeedLinkRelation.alternate) + ?.uri ?? + article.siteLink ?? + article.feedId.base, ], iconSize: 34.0), const SizedBox(width: 12.0), Expanded( diff --git a/app/lib/features/web_feed/utils/feed_parser.dart b/app/lib/features/web_feed/utils/feed_parser.dart index 0b29a8f6..7792cb3f 100644 --- a/app/lib/features/web_feed/utils/feed_parser.dart +++ b/app/lib/features/web_feed/utils/feed_parser.dart @@ -1,5 +1,4 @@ import 'package:collection/collection.dart'; -import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; import 'package:lensai/extensions/nullable.dart'; import 'package:lensai/features/web_feed/data/database/database.dart'; import 'package:lensai/features/web_feed/data/models/feed_article.dart'; @@ -79,28 +78,11 @@ class FeedParser { } } - Future> readArticles() async { + List readArticles() { final fetchDate = DateTime.now(); switch (_feed) { case final Rss1Feed feed: - final processedContents = - await GeckoBrowserExtensionService.turndownHtml( - feed.items.map((item) => item.content?.value ?? '').toList(), - ); - - final processedSummaries = - await GeckoBrowserExtensionService.turndownHtml( - feed.items - .map( - (item) => - item.description.whenNotEmpty ?? - item.dc?.description ?? - '', - ) - .toList(), - ); - return feed.items.mapIndexed((i, item) { final title = item.title.whenNotEmpty ?? item.dc?.title.whenNotEmpty; @@ -119,32 +101,19 @@ class FeedParser { authors: item.dc?.creator.whenNotEmpty.mapNotNull( (creator) => [FeedAuthor(name: creator)], ), - summaryPlain: processedSummaries[i].plain.whenNotEmpty, - summaryMarkdown: processedSummaries[i].markdown.whenNotEmpty, - links: link.mapNotNull((link) => [FeedLink(uri: link)]), + summaryHtml: + item.description.whenNotEmpty ?? + item.dc?.description.whenNotEmpty, + links: link.mapNotNull( + (link) => [ + FeedLink(uri: link, relation: FeedLinkRelation.alternate), + ], + ), tags: item.dc?.toFeedCategories(), - contentPlain: processedContents[i].plain.trim().whenNotEmpty, - contentMarkdown: processedContents[i].markdown?.trim().whenNotEmpty, + contentHtml: item.content?.value.whenNotEmpty, ); }).toList(); case final RssFeed feed: - final processedContents = - await GeckoBrowserExtensionService.turndownHtml( - feed.items.map((item) => item.content?.value ?? '').toList(), - ); - - final processedSummaries = - await GeckoBrowserExtensionService.turndownHtml( - feed.items - .map( - (item) => - item.description.whenNotEmpty ?? - item.dc?.description ?? - '', - ) - .toList(), - ); - return feed.items.mapIndexed((i, item) { final title = item.title.whenNotEmpty ?? item.dc?.title.whenNotEmpty; @@ -173,28 +142,22 @@ class FeedParser { authors: author.mapNotNull( (creator) => [FeedAuthor(name: creator)], ), - summaryPlain: processedSummaries[i].plain.whenNotEmpty, - summaryMarkdown: processedSummaries[i].markdown.whenNotEmpty, - links: link.mapNotNull((link) => [FeedLink(uri: link)]), + summaryHtml: + item.description.whenNotEmpty ?? + item.dc?.description.whenNotEmpty, + links: link.mapNotNull( + (link) => [ + FeedLink(uri: link, relation: FeedLinkRelation.alternate), + ], + ), tags: categories.isNotEmpty ? categories : item.dc?.toFeedCategories(), - contentPlain: processedContents[i].plain.trim().whenNotEmpty, - contentMarkdown: processedContents[i].markdown?.trim().whenNotEmpty, + contentHtml: item.content?.value.whenNotEmpty, ); }).toList(); case final AtomFeed feed: - final processedContents = - await GeckoBrowserExtensionService.turndownHtml( - feed.items.map((item) => item.content ?? '').toList(), - ); - - final processedSummaries = - await GeckoBrowserExtensionService.turndownHtml( - feed.items.map((item) => item.summary ?? '').toList(), - ); - final feedLink = feed.links.toFeedLinks().getRelation( FeedLinkRelation.self, ); @@ -218,16 +181,14 @@ class FeedParser { id: uniqueId, feedId: url, title: item.title.whenNotEmpty, - summaryPlain: processedSummaries[i].plain.whenNotEmpty, - summaryMarkdown: processedSummaries[i].markdown.whenNotEmpty, + summaryHtml: item.summary.whenNotEmpty, links: itemLinks, authors: authors.isNotEmpty ? authors : null, tags: tags, fetched: fetchDate, created: published, updated: updated, - contentPlain: processedContents[i].plain.trim().whenNotEmpty, - contentMarkdown: processedContents[i].markdown?.trim().whenNotEmpty, + contentHtml: item.content.whenNotEmpty, ); }).toList(); default: diff --git a/app/lib/features/web_feed/utils/fetch_entrypoint.dart b/app/lib/features/web_feed/utils/fetch_entrypoint.dart new file mode 100644 index 00000000..18c00a94 --- /dev/null +++ b/app/lib/features/web_feed/utils/fetch_entrypoint.dart @@ -0,0 +1,35 @@ +import 'package:background_fetch/background_fetch.dart'; +import 'package:lensai/core/error_observer.dart'; +import 'package:lensai/core/logger.dart'; +import 'package:lensai/features/web_feed/presentation/controllers/fetch_articles.dart'; +import 'package:riverpod/riverpod.dart'; + +@pragma('vm:entry-point') +Future backgroundFetch(HeadlessTask task) async { + final taskId = task.taskId; + + final isTimeout = task.timeout; + if (isTimeout) { + // This task has exceeded its allowed running-time. + // You must stop what you're doing and immediately .finish(taskId) + logger.e("[BackgroundFetch] Headless task timed-out: $taskId"); + await BackgroundFetch.finish(taskId); + return; + } + + final ref = ProviderContainer(observers: const [ErrorObserver()]); + try { + await ref.read(fetchArticlesControllerProvider.notifier).fetchAllArticles(); + + logger.i('Fetched articles in background'); + } catch (e, s) { + logger.e('Failed fetching articles', error: e, stackTrace: s); + } finally { + ref.dispose(); + + // Give everything a bit of time to properly dispose + await Future.delayed(const Duration(seconds: 1)); + + await BackgroundFetch.finish(taskId); + } +} diff --git a/app/lib/main.dart b/app/lib/main.dart index 437f55b4..6fcb307d 100644 --- a/app/lib/main.dart +++ b/app/lib/main.dart @@ -1,22 +1,24 @@ +import 'package:background_fetch/background_fetch.dart'; import 'package:dynamic_color/dynamic_color.dart'; import 'package:flutter/material.dart'; import 'package:home_widget/home_widget.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:lensai/core/error_observer.dart'; +import 'package:lensai/core/logger.dart'; import 'package:lensai/core/providers/defaults.dart'; import 'package:lensai/domain/services/app_initialization.dart'; import 'package:lensai/features/user/domain/repositories/general_settings.dart'; +import 'package:lensai/features/web_feed/presentation/controllers/fetch_articles.dart'; +import 'package:lensai/features/web_feed/utils/fetch_entrypoint.dart'; import 'package:lensai/presentation/hooks/on_initialization.dart'; import 'package:lensai/presentation/main_app.dart'; void main() async { WidgetsFlutterBinding.ensureInitialized(); - await HomeWidget.setAppGroupId('bang_navigator'); + await BackgroundFetch.registerHeadlessTask(backgroundFetch); - // if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) { - // await InAppWebViewController.setWebContentsDebuggingEnabled(kDebugMode); - // } + await HomeWidget.setAppGroupId('bang_navigator'); runApp( ProviderScope( @@ -33,6 +35,29 @@ void main() async { await ref .read(appInitializationServiceProvider.notifier) .initialize(); + + await BackgroundFetch.configure( + BackgroundFetchConfig( + minimumFetchInterval: 15, + enableHeadless: true, + stopOnTerminate: false, + requiredNetworkType: NetworkType.ANY, + startOnBoot: true, + ), + (String taskId) async { + try { + await ref + .read(fetchArticlesControllerProvider.notifier) + .fetchAllArticles(); + + logger.i('Fetched articles in foreground'); + } catch (e, s) { + logger.e('Failed fetching articles', error: e, stackTrace: s); + } finally { + await BackgroundFetch.finish(taskId); + } + }, + ); }); return DynamicColorBuilder( diff --git a/app/pubspec.yaml b/app/pubspec.yaml index 6fae2bc6..6b5f0682 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -7,6 +7,7 @@ environment: sdk: '>=3.7.0 <4.0.0' dependencies: + background_fetch: ^1.3.7 collection: ^1.19.1 copy_with_extension: ^6.0.1 crypto: ^3.0.6 diff --git a/packages/flutter_mozilla_components/lib/flutter_mozilla_components.dart b/packages/flutter_mozilla_components/lib/flutter_mozilla_components.dart index da842e79..4d3bc798 100644 --- a/packages/flutter_mozilla_components/lib/flutter_mozilla_components.dart +++ b/packages/flutter_mozilla_components/lib/flutter_mozilla_components.dart @@ -3,6 +3,7 @@ export 'src/data/models/load_url_flags.dart'; export 'src/data/models/source.dart'; export 'src/domain/entities/default_selection_actions.dart'; export 'src/domain/services/gecko_addon.dart'; +export 'src/domain/services/gecko_browser_extension.dart'; export 'src/domain/services/gecko_container_proxy.dart'; export 'src/domain/services/gecko_cookie.dart'; export 'src/domain/services/gecko_delete_browser_data.dart'; @@ -18,7 +19,6 @@ export 'src/domain/services/gecko_session.dart'; export 'src/domain/services/gecko_suggestions.dart'; export 'src/domain/services/gecko_tab.dart'; export 'src/domain/services/gecko_tab_content.dart'; -export 'src/domain/services/gecko_browser_extension.dart'; export 'src/geckoview_widget.dart'; export 'src/pigeons/gecko.g.dart' show diff --git a/packages/flutter_mozilla_components/lib/src/domain/services/gecko_browser_extension.dart b/packages/flutter_mozilla_components/lib/src/domain/services/gecko_browser_extension.dart index 4888fbf8..db221573 100644 --- a/packages/flutter_mozilla_components/lib/src/domain/services/gecko_browser_extension.dart +++ b/packages/flutter_mozilla_components/lib/src/domain/services/gecko_browser_extension.dart @@ -14,9 +14,16 @@ class GeckoBrowserExtensionService extends BrowserExtensionEvents { Stream get feedRequested => _feedRequest.stream; static Future> turndownHtml( - List htmlList, - ) async { - final markdownResult = await _apiInstance.getMarkdown(htmlList); + List htmlList, { + Duration timeout = const Duration(seconds: 1), + }) async { + if (htmlList.isEmpty) { + return []; + } + + final markdownResult = await _apiInstance + .getMarkdown(htmlList) + .timeout(timeout); final results = markdownResult diff --git a/packages/sqlite3_vec/example/linux/flutter/ephemeral/.plugin_symlinks/sqlite3_flutter_libs b/packages/sqlite3_vec/example/linux/flutter/ephemeral/.plugin_symlinks/sqlite3_flutter_libs index 15896ccb..3928549b 120000 --- a/packages/sqlite3_vec/example/linux/flutter/ephemeral/.plugin_symlinks/sqlite3_flutter_libs +++ b/packages/sqlite3_vec/example/linux/flutter/ephemeral/.plugin_symlinks/sqlite3_flutter_libs @@ -1 +1 @@ -/home/fafre/.pub-cache/hosted/pub.dev/sqlite3_flutter_libs-0.5.30/ \ No newline at end of file +/home/fafre/.pub-cache/hosted/pub.dev/sqlite3_flutter_libs-0.5.31/ \ No newline at end of file