fetch website data through gecko fetch api

This commit is contained in:
Fabian Freund
2025-10-30 07:01:08 +01:00
parent 13a14463f5
commit a7d2cb7144
4 changed files with 37 additions and 87 deletions
+18 -31
View File
@@ -18,21 +18,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
import 'dart:async'; import 'dart:async';
import 'dart:convert';
import 'dart:ui'; import 'dart:ui';
import 'package:collection/collection.dart';
import 'package:exceptions/exceptions.dart'; import 'package:exceptions/exceptions.dart';
import 'package:flutter/foundation.dart'; import 'package:flutter/foundation.dart';
import 'package:flutter_mozilla_components/flutter_mozilla_components.dart'; import 'package:flutter_mozilla_components/flutter_mozilla_components.dart';
import 'package:html/dom.dart'; import 'package:html/dom.dart';
import 'package:html/parser.dart' as html_parser; import 'package:html/parser.dart' as html_parser;
import 'package:http/io_client.dart';
import 'package:nullability/nullability.dart'; import 'package:nullability/nullability.dart';
import 'package:riverpod_annotation/riverpod_annotation.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:socks5_proxy/socks_client.dart';
import 'package:universal_io/io.dart';
import 'package:weblibre/core/http_error_handler.dart'; import 'package:weblibre/core/http_error_handler.dart';
import 'package:weblibre/core/logger.dart';
import 'package:weblibre/data/models/web_page_info.dart'; import 'package:weblibre/data/models/web_page_info.dart';
import 'package:weblibre/extensions/http_encoding.dart';
import 'package:weblibre/features/geckoview/domain/entities/browser_icon.dart'; import 'package:weblibre/features/geckoview/domain/entities/browser_icon.dart';
import 'package:weblibre/features/user/domain/repositories/cache.dart'; import 'package:weblibre/features/user/domain/repositories/cache.dart';
import 'package:weblibre/features/web_feed/utils/feed_finder.dart'; import 'package:weblibre/features/web_feed/utils/feed_finder.dart';
@@ -211,54 +210,42 @@ class GenericWebsiteService extends _$GenericWebsiteService {
Future<Result<WebPageInfo>> fetchPageInfo({ Future<Result<WebPageInfo>> fetchPageInfo({
required Uri url, required Uri url,
required bool isImageRequest, required bool isImageRequest,
required int? proxyPort,
}) { }) {
return Result.fromAsync(() async { return Result.fromAsync(() async {
final result = await compute((args) async { late final Map<String, Object?> result;
final [String urlString, bool isImageRequest, int? proxyPort] = args; final client = GeckoFetchService();
final httpClient = HttpClient();
if (proxyPort != null) {
SocksTCPClient.assignToHttpClient(httpClient, [
ProxySettings(InternetAddress.loopbackIPv4, proxyPort),
]);
}
final client = IOClient(httpClient);
try { try {
final baseUri = Uri.parse(urlString);
final response = await client final response = await client
.get(baseUri) .fetch(url: url)
.timeout(const Duration(seconds: 15)); .timeout(const Duration(seconds: 15));
//When this is a request for an icon and we hit an image, directly return it //When this is a request for an icon and we hit an image, directly return it
if (isImageRequest) { if (isImageRequest) {
final contentType = response.headers['content-type']; final contentType = response.headers
.firstWhereOrNull((header) => header.key == 'content-type')
?.value;
if (contentType?.contains('image/') == true) { if (contentType?.contains('image/') == true) {
return { result = {
'imageBytes': [response.bodyBytes], 'imageBytes': [response.body],
}; };
} }
} }
final document = html_parser.parse(response.bodyUnicodeFallback); final document = html_parser.parse(utf8.decode(response.body));
final title = document.querySelector('title')?.text; final title = document.querySelector('title')?.text;
final resources = _extractIcons(baseUri, document); final resources = _extractIcons(url, document);
final feeds = await FeedFinder( final feeds = await FeedFinder(url: url, document: document).parse();
url: baseUri,
document: document,
).parse();
return { result = {
'title': title, 'title': title,
'resources': resources.map(_serializeResource).toList(), 'resources': resources.map(_serializeResource).toList(),
'feeds': feeds.map((uri) => uri.toString()).toList(), 'feeds': feeds.map((uri) => uri.toString()).toList(),
}; };
} finally { } catch (e, s) {
client.close(); logger.e('Error fetching page $url', error: e, stackTrace: s);
result = {};
} }
}, <dynamic>[url.toString(), isImageRequest, proxyPort]);
if (result['imageBytes'] case final Uint8List imageBytes) { if (result['imageBytes'] case final Uint8List imageBytes) {
return WebPageInfo( return WebPageInfo(
@@ -42,7 +42,7 @@ final class GenericWebsiteServiceProvider
} }
String _$genericWebsiteServiceHash() => String _$genericWebsiteServiceHash() =>
r'9bc32261abd76a79efb57f6d27832e949cb495a6'; r'3e2ae24911a513628cdadc52ffa81c2e5415a036';
abstract class _$GenericWebsiteService extends $Notifier<void> { abstract class _$GenericWebsiteService extends $Notifier<void> {
void build(); void build();
@@ -26,11 +26,6 @@ import 'package:weblibre/domain/services/generic_website.dart';
import 'package:weblibre/extensions/ref_cache.dart'; import 'package:weblibre/extensions/ref_cache.dart';
import 'package:weblibre/features/geckoview/domain/entities/states/tab.dart'; import 'package:weblibre/features/geckoview/domain/entities/states/tab.dart';
import 'package:weblibre/features/geckoview/domain/providers/tab_state.dart'; import 'package:weblibre/features/geckoview/domain/providers/tab_state.dart';
import 'package:weblibre/features/geckoview/features/tabs/domain/repositories/container.dart';
import 'package:weblibre/features/geckoview/features/tabs/domain/repositories/tab.dart';
import 'package:weblibre/features/tor/domain/services/tor_proxy.dart';
import 'package:weblibre/features/user/data/models/tor_settings.dart';
import 'package:weblibre/features/user/domain/repositories/tor_settings.dart';
part 'website_title.g.dart'; part 'website_title.g.dart';
@@ -109,41 +104,9 @@ Future<WebPageInfo> pageInfo(
}) async { }) async {
final link = ref.cacheFor(const Duration(minutes: 2)); final link = ref.cacheFor(const Duration(minutes: 2));
final tabState = ref.read(selectedTabStateProvider);
int? proxyPort;
if (tabState?.id != null) {
final containerId = await ref
.read(tabDataRepositoryProvider.notifier)
.getContainerTabId(tabState!.id);
final containerData = await containerId.mapNotNull(
(containerId) => ref
.read(containerRepositoryProvider.notifier)
.getContainerData(containerId),
);
final torSettings = ref.read(torSettingsWithDefaultsProvider);
if (containerData?.metadata.useProxy == true ||
(tabState.isPrivate == false &&
torSettings.proxyRegularTabsMode == TorRegularTabProxyMode.all) ||
(tabState.isPrivate == true && torSettings.proxyPrivateTabsTor)) {
proxyPort = await ref.read(torProxyServiceProvider.future);
if (proxyPort == null) {
throw Exception('Could not proxy request');
}
}
}
final result = await ref final result = await ref
.watch(genericWebsiteServiceProvider.notifier) .watch(genericWebsiteServiceProvider.notifier)
.fetchPageInfo( .fetchPageInfo(url: url, isImageRequest: isImageRequest);
url: url,
isImageRequest: isImageRequest,
proxyPort: proxyPort,
);
if (!result.isSuccess) { if (!result.isSuccess) {
link.close(); link.close();
@@ -163,7 +163,7 @@ final class PageInfoProvider
} }
} }
String _$pageInfoHash() => r'87d23da9b25c3557e7270d90c2dee6f37eaaf8e0'; String _$pageInfoHash() => r'57137d47969598d8b60629b58e3de78b4737edcb';
final class PageInfoFamily extends $Family final class PageInfoFamily extends $Family
with with