prepare for multiple apps
This commit is contained in:
Fabian Freund
@@ -0,0 +1,119 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:html/dom.dart';
|
||||
|
||||
class FeedFinder {
|
||||
final Uri url;
|
||||
final Document document;
|
||||
|
||||
late final String _base;
|
||||
|
||||
FeedFinder({required this.url, required this.document}) {
|
||||
final uri = url.removeFragment();
|
||||
_base = '${uri.scheme}://${uri.authority}';
|
||||
}
|
||||
|
||||
// Future<Set<String>> _verifyCandidates(Set<String> candidates) async {
|
||||
// final results = <String>{};
|
||||
|
||||
// final client = http.Client();
|
||||
// try {
|
||||
// for (final candidate in candidates) {
|
||||
// try {
|
||||
// await client.get(Uri.parse(candidate));
|
||||
// } catch (e) {
|
||||
// continue;
|
||||
// }
|
||||
|
||||
// results.add(candidate);
|
||||
// }
|
||||
// } finally {
|
||||
// client.close();
|
||||
// }
|
||||
|
||||
// return results;
|
||||
// }
|
||||
|
||||
void _parseBody(Set<Uri> candidates) {
|
||||
for (final a in document.querySelectorAll('a')) {
|
||||
var href = a.attributes['href'];
|
||||
if (href != null) {
|
||||
if (href.contains('rss') ||
|
||||
href.contains('xml') ||
|
||||
href.contains('feed')) {
|
||||
// Fix relative URLs
|
||||
href = href.startsWith('/') ? _base + href : href;
|
||||
href = href.endsWith('/') ? href.substring(0, href.length - 2) : href;
|
||||
|
||||
// Fix naked URLs
|
||||
href = !href.startsWith('http') ? '$_base/$href' : href;
|
||||
|
||||
if (Uri.tryParse(href) case final Uri uri) {
|
||||
candidates.add(uri);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void _parseHead(Set<Uri> candidates) {
|
||||
for (final link in document.querySelectorAll("link[rel='alternate']")) {
|
||||
final type = link.attributes['type'];
|
||||
if (type != null) {
|
||||
if (type.contains('rss') || type.contains('xml')) {
|
||||
var href = link.attributes['href'];
|
||||
if (href != null) {
|
||||
// Fix relative URLs
|
||||
href = href.startsWith('/') ? _base + href : href;
|
||||
|
||||
if (Uri.tryParse(href) case final Uri uri) {
|
||||
candidates.add(uri);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<Set<Uri>> parse({
|
||||
bool parseHead = true,
|
||||
bool parseBody = true,
|
||||
// bool verifyCandidates = true,
|
||||
}) async {
|
||||
final candidates = <Uri>{};
|
||||
|
||||
// Look for feed candidates in head
|
||||
if (parseHead) {
|
||||
_parseHead(candidates);
|
||||
}
|
||||
|
||||
// Look for feed candidates in body
|
||||
if (parseBody) {
|
||||
_parseBody(candidates);
|
||||
}
|
||||
|
||||
// Verify candidates
|
||||
// if (verifyCandidates) {
|
||||
// return _verifyCandidates(candidates);
|
||||
// }
|
||||
|
||||
return candidates;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:nullability/nullability.dart';
|
||||
import 'package:rss_dart/dart_rss.dart';
|
||||
import 'package:rss_dart/domain/rss1_feed.dart';
|
||||
import 'package:weblibre/features/web_feed/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/web_feed/data/models/feed_article.dart';
|
||||
import 'package:weblibre/features/web_feed/data/models/feed_author.dart';
|
||||
import 'package:weblibre/features/web_feed/data/models/feed_link.dart';
|
||||
import 'package:weblibre/features/web_feed/extensions/atom.dart';
|
||||
import 'package:weblibre/features/web_feed/extensions/rss.dart';
|
||||
|
||||
class FeedParser {
|
||||
final Uri url;
|
||||
late Object _feed;
|
||||
|
||||
FeedParser._(this.url, this._feed);
|
||||
|
||||
factory FeedParser.parse({required Uri url, required String xmlString}) {
|
||||
final rssVersion = WebFeed.detectRssVersion(xmlString);
|
||||
final feed = switch (rssVersion) {
|
||||
RssVersion.rss1 => Rss1Feed.parse(xmlString),
|
||||
RssVersion.rss2 => RssFeed.parse(xmlString),
|
||||
RssVersion.atom => AtomFeed.parse(xmlString),
|
||||
RssVersion.unknown => throw Error.safeToString(
|
||||
'Invalid XML String? We cannot detect RSS/Atom version.',
|
||||
),
|
||||
};
|
||||
|
||||
return FeedParser._(url, feed);
|
||||
}
|
||||
|
||||
FeedData readGeneralData() {
|
||||
switch (_feed) {
|
||||
case final Rss1Feed feed:
|
||||
return FeedData(
|
||||
url: url,
|
||||
title: feed.title.whenNotEmpty ?? feed.dc?.title,
|
||||
description: feed.description.whenNotEmpty ?? feed.dc?.description,
|
||||
siteLink: feed.link.mapNotNull(Uri.tryParse),
|
||||
authors: feed.dc?.creator.whenNotEmpty.mapNotNull(
|
||||
(creator) => [FeedAuthor(name: creator)],
|
||||
),
|
||||
tags: feed.dc?.toFeedCategories(),
|
||||
);
|
||||
case final RssFeed feed:
|
||||
final categories = feed.categories.toFeedCategories();
|
||||
|
||||
return FeedData(
|
||||
url: url,
|
||||
title: feed.title.whenNotEmpty ?? feed.dc?.title,
|
||||
description: feed.description.whenNotEmpty ?? feed.dc?.description,
|
||||
siteLink: feed.link.mapNotNull(Uri.tryParse),
|
||||
authors: (feed.author.whenNotEmpty ?? feed.dc?.creator.whenNotEmpty)
|
||||
.mapNotNull((creator) => [FeedAuthor(name: creator)]),
|
||||
tags: categories.isNotEmpty
|
||||
? categories
|
||||
: feed.dc?.toFeedCategories(),
|
||||
);
|
||||
case final AtomFeed feed:
|
||||
final authors = feed.authors.toFeedAuthors();
|
||||
final tags = feed.categories.toFeedCategories();
|
||||
|
||||
return FeedData(
|
||||
url: url,
|
||||
title: feed.title.whenNotEmpty,
|
||||
icon: feed.icon.mapNotNull(Uri.tryParse),
|
||||
siteLink: feed.links
|
||||
.toFeedLinks()
|
||||
.getRelation(FeedLinkRelation.alternate)
|
||||
?.uri,
|
||||
description: feed.subtitle.whenNotEmpty,
|
||||
authors: authors.isNotEmpty ? authors : null,
|
||||
tags: tags,
|
||||
);
|
||||
default:
|
||||
throw Exception(
|
||||
'Unknown feed type in readGeneralData: ${_feed.runtimeType}',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
List<FeedArticle> readArticles() {
|
||||
final fetchDate = DateTime.now();
|
||||
|
||||
switch (_feed) {
|
||||
case final Rss1Feed feed:
|
||||
return feed.items.mapIndexed((i, item) {
|
||||
final title = item.title.whenNotEmpty ?? item.dc?.title.whenNotEmpty;
|
||||
|
||||
final itemId = item.dc?.identifier.whenNotEmpty ?? title;
|
||||
final uniqueId =
|
||||
'${item.link.whenNotEmpty ?? feed.link.whenNotEmpty ?? url}#$itemId';
|
||||
final date = SafeParseDateTime.safeParse(item.dc?.date);
|
||||
final link = item.link.mapNotNull(Uri.tryParse);
|
||||
|
||||
return FeedArticle(
|
||||
id: uniqueId,
|
||||
feedId: url,
|
||||
title: title,
|
||||
fetched: fetchDate,
|
||||
created: date,
|
||||
authors: item.dc?.creator.whenNotEmpty.mapNotNull(
|
||||
(creator) => [FeedAuthor(name: creator)],
|
||||
),
|
||||
summaryHtml:
|
||||
item.description.whenNotEmpty ??
|
||||
item.dc?.description.whenNotEmpty,
|
||||
links: link.mapNotNull(
|
||||
(link) => [
|
||||
FeedLink(uri: link, relation: FeedLinkRelation.alternate),
|
||||
],
|
||||
),
|
||||
tags: item.dc?.toFeedCategories(),
|
||||
contentHtml: item.content?.value.whenNotEmpty,
|
||||
);
|
||||
}).toList();
|
||||
case final RssFeed feed:
|
||||
return feed.items.mapIndexed((i, item) {
|
||||
final title = item.title.whenNotEmpty ?? item.dc?.title.whenNotEmpty;
|
||||
|
||||
final itemId =
|
||||
item.guid.whenNotEmpty ??
|
||||
item.dc?.identifier.whenNotEmpty ??
|
||||
title;
|
||||
|
||||
final uniqueId =
|
||||
'${item.link.whenNotEmpty ?? feed.link.whenNotEmpty ?? url}#$itemId';
|
||||
final date = SafeParseDateTime.safeParse(
|
||||
item.pubDate.whenNotEmpty ?? item.dc?.date,
|
||||
);
|
||||
final link = item.link.mapNotNull(Uri.tryParse);
|
||||
final author =
|
||||
item.author.whenNotEmpty ?? item.dc?.creator.whenNotEmpty;
|
||||
|
||||
final categories = item.categories.toFeedCategories();
|
||||
|
||||
return FeedArticle(
|
||||
id: uniqueId,
|
||||
feedId: url,
|
||||
title: title,
|
||||
fetched: fetchDate,
|
||||
created: date,
|
||||
authors: author.mapNotNull(
|
||||
(creator) => [FeedAuthor(name: creator)],
|
||||
),
|
||||
summaryHtml:
|
||||
item.description.whenNotEmpty ??
|
||||
item.dc?.description.whenNotEmpty,
|
||||
links: link.mapNotNull(
|
||||
(link) => [
|
||||
FeedLink(uri: link, relation: FeedLinkRelation.alternate),
|
||||
],
|
||||
),
|
||||
tags: categories.isNotEmpty
|
||||
? categories
|
||||
: item.dc?.toFeedCategories(),
|
||||
contentHtml: item.content?.value.whenNotEmpty,
|
||||
);
|
||||
}).toList();
|
||||
case final AtomFeed feed:
|
||||
final feedLink = feed.links.toFeedLinks().getRelation(
|
||||
FeedLinkRelation.self,
|
||||
);
|
||||
|
||||
return feed.items.mapIndexed((i, item) {
|
||||
final authors = item.authors.toFeedAuthors();
|
||||
|
||||
final tags = item.categories.toFeedCategories();
|
||||
|
||||
final itemLinks = item.links.toFeedLinks();
|
||||
final articleLink = itemLinks.getRelation(FeedLinkRelation.alternate);
|
||||
|
||||
final itemId = item.id.whenNotEmpty ?? item.title;
|
||||
final uniqueId =
|
||||
'${articleLink?.uri ?? feedLink?.uri ?? url}#$itemId';
|
||||
|
||||
final published = SafeParseDateTime.safeParse(item.published);
|
||||
final updated = SafeParseDateTime.safeParse(item.updated);
|
||||
|
||||
return FeedArticle(
|
||||
id: uniqueId,
|
||||
feedId: url,
|
||||
title: item.title.whenNotEmpty,
|
||||
summaryHtml: item.summary.whenNotEmpty,
|
||||
links: itemLinks,
|
||||
authors: authors.isNotEmpty ? authors : null,
|
||||
tags: tags,
|
||||
fetched: fetchDate,
|
||||
created: published,
|
||||
updated: updated,
|
||||
contentHtml: item.content.whenNotEmpty,
|
||||
);
|
||||
}).toList();
|
||||
default:
|
||||
throw Exception(
|
||||
'Unknown feed type in readArticles: ${_feed.runtimeType}',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:background_fetch/background_fetch.dart';
|
||||
import 'package:riverpod/riverpod.dart';
|
||||
import 'package:weblibre/core/error_observer.dart';
|
||||
import 'package:weblibre/core/logger.dart';
|
||||
import 'package:weblibre/features/web_feed/presentation/controllers/fetch_articles.dart';
|
||||
|
||||
@pragma('vm:entry-point')
|
||||
Future<void> backgroundFetch(HeadlessTask task) async {
|
||||
final taskId = task.taskId;
|
||||
|
||||
final isTimeout = task.timeout;
|
||||
if (isTimeout) {
|
||||
// This task has exceeded its allowed running-time.
|
||||
// You must stop what you're doing and immediately .finish(taskId)
|
||||
logger.e("[BackgroundFetch] Headless task timed-out: $taskId");
|
||||
await BackgroundFetch.finish(taskId);
|
||||
return;
|
||||
}
|
||||
|
||||
final ref = ProviderContainer(observers: const [ErrorObserver()]);
|
||||
try {
|
||||
await ref.read(fetchArticlesControllerProvider.notifier).fetchAllArticles();
|
||||
|
||||
logger.i('Fetched articles in background');
|
||||
} catch (e, s) {
|
||||
logger.e('Failed fetching articles', error: e, stackTrace: s);
|
||||
} finally {
|
||||
ref.dispose();
|
||||
|
||||
// Give everything a bit of time to properly dispose
|
||||
await Future.delayed(const Duration(seconds: 1));
|
||||
|
||||
await BackgroundFetch.finish(taskId);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user