From 4112779477a77db78d871d1e44df8731431bce8b Mon Sep 17 00:00:00 2001 From: Fabian Freund Date: Sun, 23 Feb 2025 17:12:46 +0100 Subject: [PATCH] feed finder --- app/lib/data/models/web_page_info.dart | 12 ++- app/lib/data/models/web_page_info.g.dart | 11 +++ app/lib/domain/services/generic_website.dart | 15 ++- .../domain/services/generic_website.g.dart | 2 +- .../geckoview/domain/entities/states/tab.dart | 6 +- app/lib/utils/feed_finder.dart | 96 +++++++++++++++++++ 6 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 app/lib/utils/feed_finder.dart diff --git a/app/lib/data/models/web_page_info.dart b/app/lib/data/models/web_page_info.dart index ac71289a..917489b7 100644 --- a/app/lib/data/models/web_page_info.dart +++ b/app/lib/data/models/web_page_info.dart @@ -1,16 +1,24 @@ import 'package:copy_with_extension/copy_with_extension.dart'; +import 'package:fast_equatable/fast_equatable.dart'; import 'package:lensai/features/geckoview/domain/entities/browser_icon.dart'; part 'web_page_info.g.dart'; @CopyWith() -class WebPageInfo { +class WebPageInfo with FastEquatable { final Uri url; final String? title; final BrowserIcon? favicon; + final Set? feeds; bool get isPageInfoComplete => (title?.isNotEmpty ?? false) && favicon != null; - WebPageInfo({required this.url, this.title, this.favicon}); + WebPageInfo({required this.url, this.title, this.favicon, this.feeds}); + + @override + bool get cacheHash => true; + + @override + List get hashParameters => [url, title, favicon, feeds]; } diff --git a/app/lib/data/models/web_page_info.g.dart b/app/lib/data/models/web_page_info.g.dart index 57a03377..5f1233b7 100644 --- a/app/lib/data/models/web_page_info.g.dart +++ b/app/lib/data/models/web_page_info.g.dart @@ -13,6 +13,8 @@ abstract class _$WebPageInfoCWProxy { WebPageInfo favicon(BrowserIcon? favicon); + WebPageInfo feeds(Set? feeds); + /// This function **does support** nullification of nullable fields. All `null` values passed to `non-nullable` fields will be ignored. You can also use `WebPageInfo(...).copyWith.fieldName(...)` to override fields one at a time with nullification support. /// /// Usage @@ -23,6 +25,7 @@ abstract class _$WebPageInfoCWProxy { Uri url, String? title, BrowserIcon? favicon, + Set? feeds, }); } @@ -41,6 +44,9 @@ class _$WebPageInfoCWProxyImpl implements _$WebPageInfoCWProxy { @override WebPageInfo favicon(BrowserIcon? favicon) => this(favicon: favicon); + @override + WebPageInfo feeds(Set? feeds) => this(feeds: feeds); + @override /// This function **does support** nullification of nullable fields. All `null` values passed to `non-nullable` fields will be ignored. You can also use `WebPageInfo(...).copyWith.fieldName(...)` to override fields one at a time with nullification support. @@ -53,6 +59,7 @@ class _$WebPageInfoCWProxyImpl implements _$WebPageInfoCWProxy { Object? url = const $CopyWithPlaceholder(), Object? title = const $CopyWithPlaceholder(), Object? favicon = const $CopyWithPlaceholder(), + Object? feeds = const $CopyWithPlaceholder(), }) { return WebPageInfo( url: url == const $CopyWithPlaceholder() @@ -67,6 +74,10 @@ class _$WebPageInfoCWProxyImpl implements _$WebPageInfoCWProxy { ? _value.favicon // ignore: cast_nullable_to_non_nullable : favicon as BrowserIcon?, + feeds: feeds == const $CopyWithPlaceholder() + ? _value.feeds + // ignore: cast_nullable_to_non_nullable + : feeds as Set?, ); } } diff --git a/app/lib/domain/services/generic_website.dart b/app/lib/domain/services/generic_website.dart index 7853303b..52ae88d9 100644 --- a/app/lib/domain/services/generic_website.dart +++ b/app/lib/domain/services/generic_website.dart @@ -13,6 +13,8 @@ import 'package:lensai/data/models/web_page_info.dart'; import 'package:lensai/extensions/nullable.dart'; import 'package:lensai/features/geckoview/domain/entities/browser_icon.dart'; import 'package:lensai/features/user/domain/repositories/cache.dart'; +import 'package:lensai/presentation/controllers/website_title.dart'; +import 'package:lensai/utils/feed_finder.dart'; import 'package:lensai/utils/lru_cache.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; @@ -198,10 +200,15 @@ class GenericWebsiteService extends _$GenericWebsiteService { final title = document.querySelector('title')?.text; final resources = _extractIcons(baseUri, document); + final feeds = await FeedFinder( + url: baseUri, + document: document, + ).parse(verifyCandidates: false); return { 'title': title, 'resources': resources.map(_serializeResource).toList(), + 'feeds': feeds, }; } finally { client.close(); @@ -221,6 +228,7 @@ class GenericWebsiteService extends _$GenericWebsiteService { url: url, title: result['title'] as String?, favicon: favicon, + feeds: result['feeds'] as Set?, ); }, exceptionHandler: handleHttpError); } @@ -278,9 +286,10 @@ class GenericWebsiteService extends _$GenericWebsiteService { return cachedIcon; } - return fetchPageInfo( - url, - ).then((result) => result.flatMap((pageInfo) => pageInfo.favicon!).value); + final result = await ref.read(pageInfoProvider(url).future); + final favicon = result.favicon!; + + return favicon; } // Future tryUpgradeToHttps(Uri httpUri) async { diff --git a/app/lib/domain/services/generic_website.g.dart b/app/lib/domain/services/generic_website.g.dart index fea956ca..7cf324db 100644 --- a/app/lib/domain/services/generic_website.g.dart +++ b/app/lib/domain/services/generic_website.g.dart @@ -7,7 +7,7 @@ part of 'generic_website.dart'; // ************************************************************************** String _$genericWebsiteServiceHash() => - r'663759beb3ae7448f4d60339557ba11c1a3a687d'; + r'12396df7370ebc31b0a53b786e5a3e27c6d05177'; /// See also [GenericWebsiteService]. @ProviderFor(GenericWebsiteService) diff --git a/app/lib/features/geckoview/domain/entities/states/tab.dart b/app/lib/features/geckoview/domain/entities/states/tab.dart index 0aca83e6..437a37a0 100644 --- a/app/lib/features/geckoview/domain/entities/states/tab.dart +++ b/app/lib/features/geckoview/domain/entities/states/tab.dart @@ -1,5 +1,4 @@ import 'package:copy_with_extension/copy_with_extension.dart'; -import 'package:fast_equatable/fast_equatable.dart'; import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; import 'package:lensai/data/models/web_page_info.dart'; import 'package:lensai/domain/entities/equatable_image.dart'; @@ -13,7 +12,7 @@ import 'package:lensai/features/geckoview/domain/entities/states/security.dart'; part 'tab.g.dart'; @CopyWith() -class TabState extends WebPageInfo with FastEquatable { +class TabState extends WebPageInfo { @CopyWithField(immutable: true) final String id; @@ -85,10 +84,9 @@ class TabState extends WebPageInfo with FastEquatable { @override List get hashParameters => [ + ...super.hashParameters, id, contextId, - url, - title, icon, thumbnail, progress, diff --git a/app/lib/utils/feed_finder.dart b/app/lib/utils/feed_finder.dart new file mode 100644 index 00000000..66034ef8 --- /dev/null +++ b/app/lib/utils/feed_finder.dart @@ -0,0 +1,96 @@ +import 'package:html/dom.dart'; +import 'package:http/http.dart' as http; + +class FeedFinder { + final Uri url; + final Document document; + + late final String _base; + + FeedFinder({required this.url, required this.document}) { + final uri = url.removeFragment(); + _base = '${uri.scheme}://${uri.authority}'; + } + + Future> _verifyCandidates(Set candidates) async { + final results = {}; + + final client = http.Client(); + try { + for (final candidate in candidates) { + try { + await client.get(Uri.parse(candidate)); + } catch (e) { + continue; + } + + results.add(candidate); + } + } finally { + client.close(); + } + + return results; + } + + void _parseBody(Set candidates) { + for (final a in document.querySelectorAll('a')) { + var href = a.attributes['href']; + if (href != null) { + if (href.contains('rss') || + href.contains('xml') || + href.contains('feed')) { + // Fix relative URLs + href = href.startsWith('/') ? _base + href : href; + href = href.endsWith('/') ? href.substring(0, href.length - 2) : href; + + // Fix naked URLs + href = !href.startsWith('http') ? '$_base/$href' : href; + + candidates.add(href); + } + } + } + } + + void _parseHead(Set candidates) { + for (final link in document.querySelectorAll("link[rel='alternate']")) { + final type = link.attributes['type']; + if (type != null) { + if (type.contains('rss') || type.contains('xml')) { + var href = link.attributes['href']; + if (href != null) { + // Fix relative URLs + href = href.startsWith('/') ? _base + href : href; + candidates.add(href); + } + } + } + } + } + + Future> parse({ + bool parseHead = true, + bool parseBody = true, + bool verifyCandidates = true, + }) async { + final candidates = {}; + + // Look for feed candidates in head + if (parseHead) { + _parseHead(candidates); + } + + // Look for feed candidates in body + if (parseBody) { + _parseBody(candidates); + } + + // Verify candidates + if (verifyCandidates) { + return _verifyCandidates(candidates); + } + + return candidates; + } +}