web feed intermediate
This commit is contained in:
Fabian Freund
@@ -0,0 +1,95 @@
|
||||
import 'package:html/dom.dart';
|
||||
|
||||
class FeedFinder {
|
||||
final Uri url;
|
||||
final Document document;
|
||||
|
||||
late final String _base;
|
||||
|
||||
FeedFinder({required this.url, required this.document}) {
|
||||
final uri = url.removeFragment();
|
||||
_base = '${uri.scheme}://${uri.authority}';
|
||||
}
|
||||
|
||||
// Future<Set<String>> _verifyCandidates(Set<String> candidates) async {
|
||||
// final results = <String>{};
|
||||
|
||||
// final client = http.Client();
|
||||
// try {
|
||||
// for (final candidate in candidates) {
|
||||
// try {
|
||||
// await client.get(Uri.parse(candidate));
|
||||
// } catch (e) {
|
||||
// continue;
|
||||
// }
|
||||
|
||||
// results.add(candidate);
|
||||
// }
|
||||
// } finally {
|
||||
// client.close();
|
||||
// }
|
||||
|
||||
// return results;
|
||||
// }
|
||||
|
||||
void _parseBody(Set<String> candidates) {
|
||||
for (final a in document.querySelectorAll('a')) {
|
||||
var href = a.attributes['href'];
|
||||
if (href != null) {
|
||||
if (href.contains('rss') ||
|
||||
href.contains('xml') ||
|
||||
href.contains('feed')) {
|
||||
// Fix relative URLs
|
||||
href = href.startsWith('/') ? _base + href : href;
|
||||
href = href.endsWith('/') ? href.substring(0, href.length - 2) : href;
|
||||
|
||||
// Fix naked URLs
|
||||
href = !href.startsWith('http') ? '$_base/$href' : href;
|
||||
|
||||
candidates.add(href);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void _parseHead(Set<String> candidates) {
|
||||
for (final link in document.querySelectorAll("link[rel='alternate']")) {
|
||||
final type = link.attributes['type'];
|
||||
if (type != null) {
|
||||
if (type.contains('rss') || type.contains('xml')) {
|
||||
var href = link.attributes['href'];
|
||||
if (href != null) {
|
||||
// Fix relative URLs
|
||||
href = href.startsWith('/') ? _base + href : href;
|
||||
candidates.add(href);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<Set<String>> parse({
|
||||
bool parseHead = true,
|
||||
bool parseBody = true,
|
||||
// bool verifyCandidates = true,
|
||||
}) async {
|
||||
final candidates = <String>{};
|
||||
|
||||
// Look for feed candidates in head
|
||||
if (parseHead) {
|
||||
_parseHead(candidates);
|
||||
}
|
||||
|
||||
// Look for feed candidates in body
|
||||
if (parseBody) {
|
||||
_parseBody(candidates);
|
||||
}
|
||||
|
||||
// Verify candidates
|
||||
// if (verifyCandidates) {
|
||||
// return _verifyCandidates(candidates);
|
||||
// }
|
||||
|
||||
return candidates;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:flutter_mozilla_components/flutter_mozilla_components.dart';
|
||||
import 'package:lensai/extensions/nullable.dart';
|
||||
import 'package:lensai/features/web_feed/data/database/database.dart';
|
||||
import 'package:lensai/features/web_feed/data/models/feed_article.dart';
|
||||
import 'package:lensai/features/web_feed/data/models/feed_author.dart';
|
||||
import 'package:lensai/features/web_feed/data/models/feed_link.dart';
|
||||
import 'package:lensai/features/web_feed/extensions/atom.dart';
|
||||
import 'package:lensai/features/web_feed/extensions/rss.dart';
|
||||
import 'package:rss_dart/dart_rss.dart';
|
||||
import 'package:rss_dart/domain/rss1_feed.dart';
|
||||
|
||||
class FeedParser {
|
||||
final Uri url;
|
||||
late Object _feed;
|
||||
|
||||
FeedParser._(this.url, this._feed);
|
||||
|
||||
factory FeedParser.parse({required Uri url, required String xmlString}) {
|
||||
final rssVersion = WebFeed.detectRssVersion(xmlString);
|
||||
final feed = switch (rssVersion) {
|
||||
RssVersion.rss1 => Rss1Feed.parse(xmlString),
|
||||
RssVersion.rss2 => RssFeed.parse(xmlString),
|
||||
RssVersion.atom => AtomFeed.parse(xmlString),
|
||||
RssVersion.unknown =>
|
||||
throw Error.safeToString(
|
||||
'Invalid XML String? We cannot detect RSS/Atom version.',
|
||||
),
|
||||
};
|
||||
|
||||
return FeedParser._(url, feed);
|
||||
}
|
||||
|
||||
FeedData readGeneralData() {
|
||||
switch (_feed) {
|
||||
case final Rss1Feed feed:
|
||||
return FeedData(
|
||||
url: url,
|
||||
title: feed.title.whenNotEmpty ?? feed.dc?.title,
|
||||
description: feed.description.whenNotEmpty ?? feed.dc?.description,
|
||||
authors: feed.dc?.creator.whenNotEmpty.mapNotNull(
|
||||
(creator) => [FeedAuthor(name: creator)],
|
||||
),
|
||||
tags: feed.dc?.toFeedCategories(),
|
||||
);
|
||||
case final RssFeed feed:
|
||||
final categories = feed.categories.toFeedCategories();
|
||||
|
||||
return FeedData(
|
||||
url: url,
|
||||
title: feed.title.whenNotEmpty ?? feed.dc?.title,
|
||||
description: feed.description.whenNotEmpty ?? feed.dc?.description,
|
||||
authors: (feed.author.whenNotEmpty ?? feed.dc?.creator.whenNotEmpty)
|
||||
.mapNotNull((creator) => [FeedAuthor(name: creator)]),
|
||||
tags:
|
||||
categories.isNotEmpty ? categories : feed.dc?.toFeedCategories(),
|
||||
);
|
||||
case final AtomFeed feed:
|
||||
final authors = feed.authors.toFeedAuthors();
|
||||
final tags = feed.categories.toFeedCategories();
|
||||
|
||||
return FeedData(
|
||||
url: url,
|
||||
title: feed.title.whenNotEmpty,
|
||||
description: feed.subtitle.whenNotEmpty,
|
||||
authors: authors.isNotEmpty ? authors : null,
|
||||
tags: tags,
|
||||
);
|
||||
default:
|
||||
throw Exception();
|
||||
}
|
||||
}
|
||||
|
||||
Future<List<FeedArticle>> readArticles() async {
|
||||
final fetchDate = DateTime.now();
|
||||
|
||||
switch (_feed) {
|
||||
case final Rss1Feed feed:
|
||||
final processedContents = await GeckoTurndownService().turndownHtml(
|
||||
feed.items.map((item) => item.content?.value ?? '').toList(),
|
||||
);
|
||||
|
||||
final processedSummaries = await GeckoTurndownService().turndownHtml(
|
||||
feed.items
|
||||
.map(
|
||||
(item) =>
|
||||
item.description.whenNotEmpty ?? item.dc?.description ?? '',
|
||||
)
|
||||
.toList(),
|
||||
);
|
||||
|
||||
return feed.items.mapIndexed((i, item) {
|
||||
final title = item.title.whenNotEmpty ?? item.dc?.title.whenNotEmpty;
|
||||
|
||||
final itemId = item.dc?.identifier.whenNotEmpty ?? title;
|
||||
final uniqueId =
|
||||
'${item.link.whenNotEmpty ?? feed.link.whenNotEmpty ?? url}#$itemId';
|
||||
final date = SafeParseDateTime.safeParse(item.dc?.date);
|
||||
final link = item.link.mapNotNull(Uri.tryParse);
|
||||
|
||||
return FeedArticle(
|
||||
id: uniqueId,
|
||||
feedId: url,
|
||||
title: title,
|
||||
fetched: fetchDate,
|
||||
created: date,
|
||||
authors: item.dc?.creator.whenNotEmpty.mapNotNull(
|
||||
(creator) => [FeedAuthor(name: creator)],
|
||||
),
|
||||
summaryPlain: processedSummaries[i].plain.whenNotEmpty,
|
||||
summaryMarkdown: processedSummaries[i].markdown.whenNotEmpty,
|
||||
links: link.mapNotNull((link) => [FeedLink(uri: link)]),
|
||||
tags: item.dc?.toFeedCategories(),
|
||||
contentPlain: processedContents[i].plain.trim().whenNotEmpty,
|
||||
contentMarkdown: processedContents[i].markdown?.trim().whenNotEmpty,
|
||||
);
|
||||
}).toList();
|
||||
case final RssFeed feed:
|
||||
final processedContents = await GeckoTurndownService().turndownHtml(
|
||||
feed.items.map((item) => item.content?.value ?? '').toList(),
|
||||
);
|
||||
|
||||
final processedSummaries = await GeckoTurndownService().turndownHtml(
|
||||
feed.items
|
||||
.map(
|
||||
(item) =>
|
||||
item.description.whenNotEmpty ?? item.dc?.description ?? '',
|
||||
)
|
||||
.toList(),
|
||||
);
|
||||
|
||||
return feed.items.mapIndexed((i, item) {
|
||||
final title = item.title.whenNotEmpty ?? item.dc?.title.whenNotEmpty;
|
||||
|
||||
final itemId =
|
||||
item.guid.whenNotEmpty ??
|
||||
item.dc?.identifier.whenNotEmpty ??
|
||||
title;
|
||||
|
||||
final uniqueId =
|
||||
'${item.link.whenNotEmpty ?? feed.link.whenNotEmpty ?? url}#$itemId';
|
||||
final date = SafeParseDateTime.safeParse(
|
||||
item.pubDate.whenNotEmpty ?? item.dc?.date,
|
||||
);
|
||||
final link = item.link.mapNotNull(Uri.tryParse);
|
||||
final author =
|
||||
item.author.whenNotEmpty ?? item.dc?.creator.whenNotEmpty;
|
||||
|
||||
final categories = item.categories.toFeedCategories();
|
||||
|
||||
return FeedArticle(
|
||||
id: uniqueId,
|
||||
feedId: url,
|
||||
title: title,
|
||||
fetched: fetchDate,
|
||||
created: date,
|
||||
authors: author.mapNotNull(
|
||||
(creator) => [FeedAuthor(name: creator)],
|
||||
),
|
||||
summaryPlain: processedSummaries[i].plain.whenNotEmpty,
|
||||
summaryMarkdown: processedSummaries[i].markdown.whenNotEmpty,
|
||||
links: link.mapNotNull((link) => [FeedLink(uri: link)]),
|
||||
tags:
|
||||
categories.isNotEmpty
|
||||
? categories
|
||||
: item.dc?.toFeedCategories(),
|
||||
contentPlain: processedContents[i].plain.trim().whenNotEmpty,
|
||||
contentMarkdown: processedContents[i].markdown?.trim().whenNotEmpty,
|
||||
);
|
||||
}).toList();
|
||||
case final AtomFeed feed:
|
||||
final processedContents = await GeckoTurndownService().turndownHtml(
|
||||
feed.items.map((item) => item.content ?? '').toList(),
|
||||
);
|
||||
|
||||
final processedSummaries = await GeckoTurndownService().turndownHtml(
|
||||
feed.items.map((item) => item.summary ?? '').toList(),
|
||||
);
|
||||
|
||||
final feedLink = feed.links.toFeedLinks().firstWhereOrNull(
|
||||
(link) => link.relation == FeedLinkRelation.self,
|
||||
);
|
||||
|
||||
return feed.items.mapIndexed((i, item) {
|
||||
final authors = item.authors.toFeedAuthors();
|
||||
|
||||
final tags = item.categories.toFeedCategories();
|
||||
|
||||
final itemLinks = item.links.toFeedLinks();
|
||||
final articleLink = itemLinks.firstWhereOrNull(
|
||||
(link) => link.relation == FeedLinkRelation.alternate,
|
||||
);
|
||||
|
||||
final itemId = item.id.whenNotEmpty ?? item.title;
|
||||
final uniqueId =
|
||||
'${articleLink?.uri ?? feedLink?.uri ?? url}#$itemId';
|
||||
|
||||
final published = SafeParseDateTime.safeParse(item.published);
|
||||
final updated = SafeParseDateTime.safeParse(item.updated);
|
||||
|
||||
return FeedArticle(
|
||||
id: uniqueId,
|
||||
feedId: url,
|
||||
title: item.title.whenNotEmpty,
|
||||
summaryPlain: processedSummaries[i].plain.whenNotEmpty,
|
||||
summaryMarkdown: processedSummaries[i].markdown.whenNotEmpty,
|
||||
links: itemLinks,
|
||||
authors: authors.isNotEmpty ? authors : null,
|
||||
tags: tags,
|
||||
fetched: fetchDate,
|
||||
created: published,
|
||||
updated: updated,
|
||||
contentPlain: processedContents[i].plain.trim().whenNotEmpty,
|
||||
contentMarkdown: processedContents[i].markdown?.trim().whenNotEmpty,
|
||||
);
|
||||
}).toList();
|
||||
default:
|
||||
throw Exception();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user