From a7d2cb7144b9f846c76f508cf1956d661809991e Mon Sep 17 00:00:00 2001 From: Fabian Freund Date: Thu, 30 Oct 2025 07:01:08 +0100 Subject: [PATCH] fetch website data through gecko fetch api --- app/lib/domain/services/generic_website.dart | 81 ++++++++----------- .../domain/services/generic_website.g.dart | 2 +- .../controllers/website_title.dart | 39 +-------- .../controllers/website_title.g.dart | 2 +- 4 files changed, 37 insertions(+), 87 deletions(-) diff --git a/app/lib/domain/services/generic_website.dart b/app/lib/domain/services/generic_website.dart index 15e225f9..175edb94 100644 --- a/app/lib/domain/services/generic_website.dart +++ b/app/lib/domain/services/generic_website.dart @@ -18,21 +18,20 @@ * along with this program. If not, see . */ import 'dart:async'; +import 'dart:convert'; import 'dart:ui'; +import 'package:collection/collection.dart'; import 'package:exceptions/exceptions.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; import 'package:html/dom.dart'; import 'package:html/parser.dart' as html_parser; -import 'package:http/io_client.dart'; import 'package:nullability/nullability.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:socks5_proxy/socks_client.dart'; -import 'package:universal_io/io.dart'; import 'package:weblibre/core/http_error_handler.dart'; +import 'package:weblibre/core/logger.dart'; import 'package:weblibre/data/models/web_page_info.dart'; -import 'package:weblibre/extensions/http_encoding.dart'; import 'package:weblibre/features/geckoview/domain/entities/browser_icon.dart'; import 'package:weblibre/features/user/domain/repositories/cache.dart'; import 'package:weblibre/features/web_feed/utils/feed_finder.dart'; @@ -211,54 +210,42 @@ class GenericWebsiteService extends _$GenericWebsiteService { Future> fetchPageInfo({ required Uri url, required bool isImageRequest, - required int? proxyPort, }) { return Result.fromAsync(() async { - final result = await compute((args) async { - final [String urlString, bool isImageRequest, int? proxyPort] = args; + late final Map result; + final client = GeckoFetchService(); + try { + final response = await client + .fetch(url: url) + .timeout(const Duration(seconds: 15)); - final httpClient = HttpClient(); - if (proxyPort != null) { - SocksTCPClient.assignToHttpClient(httpClient, [ - ProxySettings(InternetAddress.loopbackIPv4, proxyPort), - ]); - } - - final client = IOClient(httpClient); - try { - final baseUri = Uri.parse(urlString); - final response = await client - .get(baseUri) - .timeout(const Duration(seconds: 15)); - - //When this is a request for an icon and we hit an image, directly return it - if (isImageRequest) { - final contentType = response.headers['content-type']; - if (contentType?.contains('image/') == true) { - return { - 'imageBytes': [response.bodyBytes], - }; - } + //When this is a request for an icon and we hit an image, directly return it + if (isImageRequest) { + final contentType = response.headers + .firstWhereOrNull((header) => header.key == 'content-type') + ?.value; + if (contentType?.contains('image/') == true) { + result = { + 'imageBytes': [response.body], + }; } - - final document = html_parser.parse(response.bodyUnicodeFallback); - - final title = document.querySelector('title')?.text; - final resources = _extractIcons(baseUri, document); - final feeds = await FeedFinder( - url: baseUri, - document: document, - ).parse(); - - return { - 'title': title, - 'resources': resources.map(_serializeResource).toList(), - 'feeds': feeds.map((uri) => uri.toString()).toList(), - }; - } finally { - client.close(); } - }, [url.toString(), isImageRequest, proxyPort]); + + final document = html_parser.parse(utf8.decode(response.body)); + + final title = document.querySelector('title')?.text; + final resources = _extractIcons(url, document); + final feeds = await FeedFinder(url: url, document: document).parse(); + + result = { + 'title': title, + 'resources': resources.map(_serializeResource).toList(), + 'feeds': feeds.map((uri) => uri.toString()).toList(), + }; + } catch (e, s) { + logger.e('Error fetching page $url', error: e, stackTrace: s); + result = {}; + } if (result['imageBytes'] case final Uint8List imageBytes) { return WebPageInfo( diff --git a/app/lib/domain/services/generic_website.g.dart b/app/lib/domain/services/generic_website.g.dart index 2b556015..1d095957 100644 --- a/app/lib/domain/services/generic_website.g.dart +++ b/app/lib/domain/services/generic_website.g.dart @@ -42,7 +42,7 @@ final class GenericWebsiteServiceProvider } String _$genericWebsiteServiceHash() => - r'9bc32261abd76a79efb57f6d27832e949cb495a6'; + r'3e2ae24911a513628cdadc52ffa81c2e5415a036'; abstract class _$GenericWebsiteService extends $Notifier { void build(); diff --git a/app/lib/presentation/controllers/website_title.dart b/app/lib/presentation/controllers/website_title.dart index 3b6dfb3a..69e4a614 100644 --- a/app/lib/presentation/controllers/website_title.dart +++ b/app/lib/presentation/controllers/website_title.dart @@ -26,11 +26,6 @@ import 'package:weblibre/domain/services/generic_website.dart'; import 'package:weblibre/extensions/ref_cache.dart'; import 'package:weblibre/features/geckoview/domain/entities/states/tab.dart'; import 'package:weblibre/features/geckoview/domain/providers/tab_state.dart'; -import 'package:weblibre/features/geckoview/features/tabs/domain/repositories/container.dart'; -import 'package:weblibre/features/geckoview/features/tabs/domain/repositories/tab.dart'; -import 'package:weblibre/features/tor/domain/services/tor_proxy.dart'; -import 'package:weblibre/features/user/data/models/tor_settings.dart'; -import 'package:weblibre/features/user/domain/repositories/tor_settings.dart'; part 'website_title.g.dart'; @@ -109,41 +104,9 @@ Future pageInfo( }) async { final link = ref.cacheFor(const Duration(minutes: 2)); - final tabState = ref.read(selectedTabStateProvider); - - int? proxyPort; - if (tabState?.id != null) { - final containerId = await ref - .read(tabDataRepositoryProvider.notifier) - .getContainerTabId(tabState!.id); - - final containerData = await containerId.mapNotNull( - (containerId) => ref - .read(containerRepositoryProvider.notifier) - .getContainerData(containerId), - ); - - final torSettings = ref.read(torSettingsWithDefaultsProvider); - - if (containerData?.metadata.useProxy == true || - (tabState.isPrivate == false && - torSettings.proxyRegularTabsMode == TorRegularTabProxyMode.all) || - (tabState.isPrivate == true && torSettings.proxyPrivateTabsTor)) { - proxyPort = await ref.read(torProxyServiceProvider.future); - - if (proxyPort == null) { - throw Exception('Could not proxy request'); - } - } - } - final result = await ref .watch(genericWebsiteServiceProvider.notifier) - .fetchPageInfo( - url: url, - isImageRequest: isImageRequest, - proxyPort: proxyPort, - ); + .fetchPageInfo(url: url, isImageRequest: isImageRequest); if (!result.isSuccess) { link.close(); diff --git a/app/lib/presentation/controllers/website_title.g.dart b/app/lib/presentation/controllers/website_title.g.dart index 33fb2a86..465af7ff 100644 --- a/app/lib/presentation/controllers/website_title.g.dart +++ b/app/lib/presentation/controllers/website_title.g.dart @@ -163,7 +163,7 @@ final class PageInfoProvider } } -String _$pageInfoHash() => r'87d23da9b25c3557e7270d90c2dee6f37eaaf8e0'; +String _$pageInfoHash() => r'57137d47969598d8b60629b58e3de78b4737edcb'; final class PageInfoFamily extends $Family with