From 03c7081cb27cbf0d83768f956f07725f8044a938 Mon Sep 17 00:00:00 2001
From: Fabian Freund <github@fabi.online>
Date: Mon, 11 Aug 2025 11:38:54 +0200
Subject: [PATCH] fallback utf8 instead latin when charset missing

---
 app/lib/domain/services/generic_website.dart  |  3 +-
 app/lib/extensions/http_encoding.dart         | 39 +++++++++++++++++++
 .../web_feed/domain/services/feed_reader.dart |  6 ++-
 3 files changed, 46 insertions(+), 2 deletions(-)
 create mode 100644 app/lib/extensions/http_encoding.dart

diff --git a/app/lib/domain/services/generic_website.dart b/app/lib/domain/services/generic_website.dart
index 9a5e0608..48b394ae 100644
--- a/app/lib/domain/services/generic_website.dart
+++ b/app/lib/domain/services/generic_website.dart
@@ -32,6 +32,7 @@ import 'package:socks5_proxy/socks_client.dart';
 import 'package:universal_io/io.dart';
 import 'package:weblibre/core/http_error_handler.dart';
 import 'package:weblibre/data/models/web_page_info.dart';
+import 'package:weblibre/extensions/http_encoding.dart';
 import 'package:weblibre/features/geckoview/domain/entities/browser_icon.dart';
 import 'package:weblibre/features/user/domain/repositories/cache.dart';
 import 'package:weblibre/features/web_feed/utils/feed_finder.dart';
@@ -240,7 +241,7 @@ class GenericWebsiteService extends _$GenericWebsiteService {
             }
           }
 
-          final document = html_parser.parse(response.body);
+          final document = html_parser.parse(response.bodyUnicodeFallback);
 
           final title = document.querySelector('title')?.text;
           final resources = _extractIcons(baseUri, document);
diff --git a/app/lib/extensions/http_encoding.dart b/app/lib/extensions/http_encoding.dart
new file mode 100644
index 00000000..c5acfc93
--- /dev/null
+++ b/app/lib/extensions/http_encoding.dart
@@ -0,0 +1,39 @@
+import 'dart:convert';
+
+import 'package:http/http.dart';
+import 'package:http_parser/http_parser.dart';
+
+/// This is modified from default latin1 fallback to use utf8
+Encoding _encodingForHeaders(Map<String, String> headers) =>
+    _encodingForContentTypeHeader(_contentTypeForHeaders(headers), utf8);
+
+/// Returns the [MediaType] object for the given headers' content-type.
+///
+/// Defaults to `application/octet-stream`.
+MediaType _contentTypeForHeaders(Map<String, String> headers) {
+  final contentType = headers['content-type'];
+  if (contentType != null) return MediaType.parse(contentType);
+  return MediaType('application', 'octet-stream');
+}
+
+Encoding _encodingForContentTypeHeader(
+  MediaType contentTypeHeader, [
+  Encoding fallback = latin1,
+]) {
+  final charset = contentTypeHeader.parameters['charset'];
+
+  // Default to utf8 for application/json when charset is unspecified.
+  if (contentTypeHeader.type == 'application' &&
+      contentTypeHeader.subtype == 'json' &&
+      charset == null) {
+    return utf8;
+  }
+
+  // Attempt to find the encoding or fall back to the default.
+  return charset != null ? Encoding.getByName(charset) ?? fallback : fallback;
+}
+
+extension ResponesEncoding on Response {
+  String get bodyUnicodeFallback =>
+      _encodingForHeaders(headers).decode(bodyBytes);
+}
diff --git a/app/lib/features/web_feed/domain/services/feed_reader.dart b/app/lib/features/web_feed/domain/services/feed_reader.dart
index cf779616..db4d4ecc 100644
--- a/app/lib/features/web_feed/domain/services/feed_reader.dart
+++ b/app/lib/features/web_feed/domain/services/feed_reader.dart
@@ -21,6 +21,7 @@ import 'package:flutter/foundation.dart';
 import 'package:flutter/services.dart';
 import 'package:http/http.dart' as http;
 import 'package:riverpod_annotation/riverpod_annotation.dart';
+import 'package:weblibre/extensions/http_encoding.dart';
 import 'package:weblibre/features/web_feed/data/models/feed_parse_result.dart';
 import 'package:weblibre/features/web_feed/utils/feed_parser.dart';
 
@@ -44,7 +45,10 @@ class FeedReader extends _$FeedReader {
             .get(url)
             .timeout(const Duration(seconds: 30));
 
-        final parser = FeedParser.parse(url: url, xmlString: response.body);
+        final parser = FeedParser.parse(
+          url: url,
+          xmlString: response.bodyUnicodeFallback,
+        );
         final result = FeedParseResult(
           feedData: parser.readGeneralData(),
           articleData: parser.readArticles(),