web feed intermediate

app/lib/features/web_feed/data/database/converters/feed_authors.dart

@@ -0,0 +1,21 @@
+import 'dart:convert';
+
+import 'package:drift/drift.dart';
+import 'package:lensai/features/web_feed/data/models/feed_author.dart';
+
+class FeedAuthorsConverter extends TypeConverter<List<FeedAuthor>, String> {
+  const FeedAuthorsConverter();
+
+  @override
+  List<FeedAuthor> fromSql(String fromDb) {
+    final authors = jsonDecode(fromDb) as List<dynamic>;
+    return authors
+        .map((author) => FeedAuthor.fromJson(author as Map<String, dynamic>))
+        .toList();
+  }
+
+  @override
+  String toSql(List<FeedAuthor> value) {
+    return jsonEncode(value.map((author) => author.toJson()).toList());
+  }
+}

app/lib/features/web_feed/data/database/converters/feed_categories.dart

@@ -0,0 +1,24 @@
+import 'dart:convert';
+
+import 'package:drift/drift.dart';
+import 'package:lensai/features/web_feed/data/models/feed_category.dart';
+
+class FeedCategoriesConverter
+    extends TypeConverter<List<FeedCategory>, String> {
+  const FeedCategoriesConverter();
+
+  @override
+  List<FeedCategory> fromSql(String fromDb) {
+    final categories = jsonDecode(fromDb) as List<dynamic>;
+    return categories
+        .map(
+          (category) => FeedCategory.fromJson(category as Map<String, dynamic>),
+        )
+        .toList();
+  }
+
+  @override
+  String toSql(List<FeedCategory> value) {
+    return jsonEncode(value.map((category) => category.toJson()).toList());
+  }
+}

app/lib/features/web_feed/data/database/converters/feed_data.dart

@@ -0,0 +1,16 @@
+import 'package:json_annotation/json_annotation.dart';
+import 'package:lensai/features/web_feed/data/database/database.dart';
+
+class FeedDataConverter extends JsonConverter<FeedData, Map<String, dynamic>> {
+  const FeedDataConverter();
+
+  @override
+  FeedData fromJson(Map<String, dynamic> json) {
+    return FeedData.fromJson(json);
+  }
+
+  @override
+  Map<String, dynamic> toJson(FeedData object) {
+    return object.toJson();
+  }
+}

app/lib/features/web_feed/data/database/converters/feed_links.dart

@@ -0,0 +1,21 @@
+import 'dart:convert';
+
+import 'package:drift/drift.dart';
+import 'package:lensai/features/web_feed/data/models/feed_link.dart';
+
+class FeedLinksConverter extends TypeConverter<List<FeedLink>, String> {
+  const FeedLinksConverter();
+
+  @override
+  List<FeedLink> fromSql(String fromDb) {
+    final links = jsonDecode(fromDb) as List<dynamic>;
+    return links
+        .map((link) => FeedLink.fromJson(link as Map<String, dynamic>))
+        .toList();
+  }
+
+  @override
+  String toSql(List<FeedLink> value) {
+    return jsonEncode(value.map((link) => link.toJson()).toList());
+  }
+}

app/lib/features/web_feed/data/database/daos/article.dart

@@ -0,0 +1,119 @@
+import 'package:drift/drift.dart';
+import 'package:lensai/features/web_feed/data/database/database.dart';
+import 'package:lensai/features/web_feed/data/models/feed_article.dart';
+import 'package:lensai/features/web_feed/data/models/feed_article_query_result.dart';
+
+part 'article.g.dart';
+
+@DriftAccessor()
+class ArticleDao extends DatabaseAccessor<FeedDatabase> with _$ArticleDaoMixin {
+  ArticleDao(super.attachedDatabase);
+
+  Selectable<FeedArticle> getFeedArticles(Uri? url) {
+    final select = db.article.select();
+
+    if (url != null) {
+      select.where((article) => article.feedId.equalsValue(url));
+    }
+
+    return select..orderBy([
+      (row) => OrderingTerm(
+        expression: coalesce([row.updated, row.created]),
+        mode: OrderingMode.desc,
+      ),
+    ]);
+  }
+
+  SingleOrNullSelectable<FeedArticle> getArticleById(String articleId) {
+    return db.article.select()..where((row) => row.id.equals(articleId));
+  }
+
+  Future<void> upsertArticles(List<FeedArticle> articles) {
+    return db.transaction(() async {
+      await Future.wait(
+        articles
+            .map(
+              (article) => db.article.insertOne(
+                article,
+                onConflict: DoUpdate(
+                  (old) {
+                    return ArticleCompanion(
+                      authors: Value(article.authors),
+                      contentMarkdown: Value(article.contentMarkdown),
+                      contentPlain: Value(article.contentPlain),
+                      links: Value(article.links),
+                      summaryMarkdown: Value(article.summaryMarkdown),
+                      summaryPlain: Value(article.summaryPlain),
+                      tags: Value(article.tags),
+                      title: Value(article.title),
+                      updated: Value(article.updated),
+                    );
+                  },
+                  where:
+                      (old) =>
+                          old.updated.isNotNull() &
+                          old.updated.isSmallerThanValue(
+                            article.updated ?? DateTime(0),
+                          ),
+                ),
+              ),
+            )
+            .toList(),
+      );
+    });
+  }
+
+  Future<int> updateArticleRead(String articleId, DateTime? read) {
+    final statement =
+        db.article.update()..where((article) => article.id.equals(articleId));
+
+    return statement.write(ArticleCompanion(lastRead: Value(read)));
+  }
+
+  Selectable<(String, int)> getUnreadArticleCount() {
+    final count = countAll();
+
+    final countByFeed =
+        db.article.selectOnly()
+          ..addColumns([db.article.feedId, count])
+          ..where(
+            db.article.lastRead.isNull() |
+                (db.article.updated.isNotNull() &
+                    db.article.lastRead.isSmallerThan(db.article.lastRead)),
+          )
+          ..groupBy([db.article.feedId]);
+
+    return countByFeed.map(
+      (result) => (result.read(db.article.feedId)!, result.read(count)!),
+    );
+  }
+
+  Selectable<FeedArticleQueryResult> queryArticles({
+    required String matchPrefix,
+    required String matchSuffix,
+    required String ellipsis,
+    required int snippetLength,
+    required String searchString,
+    required Uri? feedId,
+  }) {
+    final ftsQuery = db.buildFtsQuery(searchString);
+
+    if (ftsQuery.isNotEmpty) {
+      return db.queryArticlesFullContent(
+        feedId: feedId?.toString(),
+        query: ftsQuery,
+        snippetLength: snippetLength,
+        beforeMatch: matchPrefix,
+        afterMatch: matchSuffix,
+        ellipsis: ellipsis,
+      );
+    } else {
+      return db.queryArticlesBasic(
+        feedId: feedId?.toString(),
+        query: db.buildLikeQuery(searchString),
+        beforeMatch: matchPrefix,
+        afterMatch: matchSuffix,
+      );
+    }
+  }
+}

app/lib/features/web_feed/data/database/daos/article.g.dart

@@ -0,0 +1,6 @@
+// GENERATED CODE - DO NOT MODIFY BY HAND
+
+part of 'article.dart';
+
+// ignore_for_file: type=lint
+mixin _$ArticleDaoMixin on DatabaseAccessor<FeedDatabase> {}

app/lib/features/web_feed/data/database/daos/feed.dart

@@ -0,0 +1,38 @@
+import 'package:drift/drift.dart';
+import 'package:lensai/features/web_feed/data/database/database.dart';
+
+part 'feed.g.dart';
+
+@DriftAccessor()
+class FeedDao extends DatabaseAccessor<FeedDatabase> with _$FeedDaoMixin {
+  FeedDao(super.attachedDatabase);
+
+  Selectable<FeedData> getFeeds() {
+    return db.feed.select();
+  }
+
+  Future<int> updateFeedFetched(Uri url, DateTime fetched) {
+    final statement =
+        db.feed.update()..where((feed) => feed.url.equalsValue(url));
+
+    return statement.write(FeedCompanion(lastFetched: Value(fetched)));
+  }
+
+  Future<int> deleteFeed(Uri url) {
+    return db.feed.deleteWhere((feed) => feed.url.equals(url.toString()));
+  }
+
+  Future<int> upsertFeed(FeedData feedData) {
+    return db.feed.insertOne(
+      feedData,
+      onConflict: DoUpdate((old) {
+        return FeedCompanion(
+          authors: Value(feedData.authors),
+          title: Value(feedData.title),
+          description: Value(feedData.description),
+          tags: Value(feedData.tags),
+        );
+      }),
+    );
+  }
+}

app/lib/features/web_feed/data/database/daos/feed.g.dart

@@ -0,0 +1,6 @@
+// GENERATED CODE - DO NOT MODIFY BY HAND
+
+part of 'feed.dart';
+
+// ignore_for_file: type=lint
+mixin _$FeedDaoMixin on DatabaseAccessor<FeedDatabase> {}

app/lib/features/web_feed/data/database/database.dart

@@ -0,0 +1,36 @@
+import 'package:drift/drift.dart';
+import 'package:lensai/data/database/converters/uri.dart';
+import 'package:lensai/features/search/domain/fts_tokenizer.dart';
+import 'package:lensai/features/web_feed/data/database/converters/feed_authors.dart';
+import 'package:lensai/features/web_feed/data/database/converters/feed_categories.dart';
+import 'package:lensai/features/web_feed/data/database/converters/feed_links.dart';
+import 'package:lensai/features/web_feed/data/database/daos/article.dart';
+import 'package:lensai/features/web_feed/data/database/daos/feed.dart';
+import 'package:lensai/features/web_feed/data/models/feed_article.dart';
+import 'package:lensai/features/web_feed/data/models/feed_article_query_result.dart';
+import 'package:lensai/features/web_feed/data/models/feed_author.dart';
+import 'package:lensai/features/web_feed/data/models/feed_category.dart';
+import 'package:lensai/features/web_feed/data/models/feed_link.dart';
+
+part 'database.g.dart';
+
+@DriftDatabase(include: {'database.drift'}, daos: [ArticleDao, FeedDao])
+class FeedDatabase extends _$FeedDatabase with TrigramQueryBuilderMixin {
+  @override
+  final int schemaVersion = 1;
+
+  @override
+  final int ftsTokenLimit = 10;
+  @override
+  final int ftsMinTokenLength = 3;
+
+  @override
+  MigrationStrategy get migration => MigrationStrategy(
+    beforeOpen: (details) async {
+      await customStatement('PRAGMA foreign_keys = ON;');
+      await optimizeFtsIndex();
+    },
+  );
+
+  FeedDatabase(super.e);
+}

app/lib/features/web_feed/data/database/database.drift

@@ -0,0 +1,116 @@
+import 'package:lensai/data/database/converters/uri.dart';
+import 'package:lensai/features/web_feed/data/models/feed_article_query_result.dart';
+import 'package:lensai/features/web_feed/data/database/converters/feed_authors.dart';
+import 'package:lensai/features/web_feed/data/database/converters/feed_categories.dart';
+import 'package:lensai/features/web_feed/data/database/converters/feed_links.dart';
+import 'package:lensai/features/web_feed/data/models/feed_article.dart';
+
+CREATE TABLE feed (
+  url TEXT PRIMARY KEY NOT NULL MAPPED BY `const UriConverter()`,
+  title TEXT,
+  description TEXT,
+  authors TEXT MAPPED BY `const FeedAuthorsConverter()`,
+  tags TEXT MAPPED BY `const FeedCategoriesConverter()`,
+  last_fetched DATETIME
+);
+
+CREATE TABLE article (
+  id TEXT PRIMARY KEY NOT NULL,
+  feed_id TEXT NOT NULL MAPPED BY `const UriConverter()` REFERENCES feed ("url") ON DELETE CASCADE,
+  fetched DATETIME NOT NULL,
+  created DATETIME,
+  updated DATETIME,
+  last_read DATETIME, 
+  title TEXT,
+  authors TEXT MAPPED BY `const FeedAuthorsConverter()`,
+  tags TEXT MAPPED BY `const FeedCategoriesConverter()`,
+  links TEXT MAPPED BY `const FeedLinksConverter()`,
+  summaryMarkdown TEXT,
+  summaryPlain TEXT,
+  contentMarkdown TEXT,
+  contentPlain TEXT
+) WITH FeedArticle;
+
+CREATE INDEX article_feed_id ON article (feed_id);
+
+CREATE VIRTUAL TABLE article_fts 
+  USING fts5(
+    title,
+    summaryPlain,
+    contentPlain,
+    content=article,
+    tokenize="trigram"
+  );
+
+-- Triggers to keep the FTS index up to date.
+CREATE TRIGGER article_after_insert AFTER INSERT ON article BEGIN
+  INSERT INTO 
+    article_fts(rowid, title, summaryPlain, contentPlain) 
+    VALUES (new.rowid, new.title, new.summaryPlain, new.contentPlain);
+END;
+CREATE TRIGGER article_after_delete AFTER DELETE ON article BEGIN
+  INSERT INTO 
+    article_fts(article_fts, rowid, title, summaryPlain, contentPlain) 
+    VALUES('delete', old.rowid, old.title, old.summaryPlain, old.contentPlain);
+END;
+CREATE TRIGGER article_after_update AFTER UPDATE ON article BEGIN
+  INSERT INTO 
+    article_fts(article_fts, rowid, title, summaryPlain, contentPlain) 
+    VALUES('delete', old.rowid, old.title, old.summaryPlain, old.contentPlain);
+  INSERT INTO 
+    article_fts(rowid, title, summaryPlain, contentPlain) 
+    VALUES (new.rowid, new.title, new.summaryPlain, new.contentPlain);
+END;
+
+optimizeFtsIndex:
+  INSERT INTO article_fts(article_fts) VALUES ('optimize');
+
+queryArticlesBasic(:feed_id AS TEXT OR NULL) WITH FeedArticleQueryResult:
+  WITH weights AS (
+  SELECT 
+    -- Customize these weights (higher = more important)
+    1.0 as title_weight    -- Title matches are most important
+  )
+  SELECT 
+      a.*,
+      highlight(article_fts, 0, :beforeMatch, :afterMatch) AS title,
+      (
+        bm25(article_fts, weights.title_weight)
+      ) AS weighted_rank
+    FROM article_fts(:query) fts
+    INNER JOIN
+      article a ON a.rowid = fts.rowid
+    CROSS JOIN weights
+    WHERE
+      fts.title LIKE :query AND
+      :feed_id IS NULL OR a.feed_id = :feed_id
+    ORDER BY 
+      weighted_rank ASC,
+      a.created DESC NULLS LAST;
+
+queryArticlesFullContent(:feed_id AS TEXT OR NULL) WITH FeedArticleQueryResult:
+  WITH weights AS (
+  SELECT 
+    -- Customize these weights (higher = more important)
+    5.0 as title_weight,    -- Title matches are most important
+    2.0 as summary_weight,  -- Summary matches are quite important
+    1.0 as content_weight   -- Content matches are basic
+  )
+  SELECT 
+      a.*,
+      highlight(article_fts, 0, :beforeMatch, :afterMatch) AS title,
+      snippet(article_fts, 1, :beforeMatch, :afterMatch, :ellipsis, :snippetLength) AS summary,
+      snippet(article_fts, 2, :beforeMatch, :afterMatch, :ellipsis, :snippetLength) AS content,
+      (
+        bm25(article_fts, weights.title_weight, weights.summary_weight, 
+            weights.content_weight)
+      ) AS weighted_rank
+    FROM article_fts(:query) fts
+    INNER JOIN
+      article a ON a.rowid = fts.rowid
+    CROSS JOIN weights
+    WHERE
+      :feed_id IS NULL OR a.feed_id = :feed_id
+    ORDER BY 
+      weighted_rank ASC,
+      a.created DESC NULLS LAST;