From 3cedada44e92bfd3ac1eddf8bbce7df28bec8672 Mon Sep 17 00:00:00 2001 From: Fabian Freund Date: Sun, 25 May 2025 22:41:19 +0200 Subject: [PATCH] strip chat --- app/lib/core/providers/models.dart | 40 - app/lib/core/providers/models.g.dart | 63 - .../converters/chat_metadata_converter.dart | 18 - .../data/database/daos/messages.dart | 137 -- .../data/database/daos/messages.g.dart | 6 - .../data/database/daos/metadata.dart | 27 - .../data/database/daos/metadata.g.dart | 6 - .../chat_store/data/database/database.dart | 15 - .../chat_store/data/database/database.drift | 65 - .../chat_store/data/database/database.g.dart | 871 ----------- .../chat_store/data/models/chat_metadata.dart | 20 - .../data/models/chat_metadata.g.dart | 18 - .../chat_store/data/models/message_types.dart | 22 - .../features/chat_store/data/providers.dart | 50 - .../features/chat_store/data/providers.g.dart | 26 - .../chat_store/domain/chat_memory.dart | 155 -- .../controllers/drift_chat_controller.dart | 166 --- .../features/chat_store/domain/providers.dart | 25 - .../chat_store/domain/providers.g.dart | 273 ---- .../domain/repositories/chat_message.dart | 86 -- .../domain/repositories/chat_message.g.dart | 165 --- .../domain/repositories/chat_metadata.dart | 21 - .../domain/repositories/chat_metadata.g.dart | 165 --- .../presentation/dialogs/web_page_dialog.dart | 58 - .../browser/presentation/screens/browser.dart | 7 - .../widgets/browser_modules/browser_view.dart | 2 - .../features/tabs/data/database/database.dart | 24 +- .../tabs/data/database/database.drift | 8 - .../tabs/data/database/database.g.dart | 1301 ----------------- .../features/tabs/data/providers.dart | 7 - .../features/tabs/data/providers.g.dart | 2 +- .../features/chat/domain/chat_backend.dart | 79 - .../features/chat/domain/chat_backend.g.dart | 161 -- .../widgets/chat_text_message.dart | 57 - .../presentation/widgets/qa_chat_input.dart | 186 --- .../presentation/widgets/tab_qa_chat.dart | 72 - .../chat/services/qa_memory_chain.dart | 59 - .../chat/services/qa_memory_chain.g.dart | 219 --- .../data/database/daos/vector.dart | 173 --- .../data/database/daos/vector.g.dart | 6 - .../vector_store/data/database/migrator.dart | 15 - .../data/database/vector_store.drift | 72 - .../data/models/vector_result.dart | 40 - .../domain/repositories/document.dart | 115 -- .../domain/repositories/document.g.dart | 28 - .../domain/sqlite_vector_store.dart | 61 - .../vector_store/utils/document_uuid.dart | 26 - .../features/vector_store/utils/hash.dart | 17 - .../utils/markdown_document_splitter.dart | 116 -- app/lib/utils/langchain_utils.dart | 12 - app/pubspec.yaml | 37 +- app/pubspec_overrides.yaml | 16 +- 52 files changed, 5 insertions(+), 5411 deletions(-) delete mode 100644 app/lib/core/providers/models.dart delete mode 100644 app/lib/core/providers/models.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/converters/chat_metadata_converter.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/daos/messages.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/daos/messages.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/daos/metadata.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/daos/metadata.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/database.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/database/database.drift delete mode 100644 app/lib/features/chat/features/chat_store/data/database/database.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/models/chat_metadata.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/models/chat_metadata.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/models/message_types.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/providers.dart delete mode 100644 app/lib/features/chat/features/chat_store/data/providers.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/chat_memory.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/controllers/drift_chat_controller.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/providers.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/providers.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/repositories/chat_message.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/repositories/chat_message.g.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.dart delete mode 100644 app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.g.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.g.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/chat_text_message.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/qa_chat_input.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/tab_qa_chat.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.g.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.g.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/data/database/migrator.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/data/database/vector_store.drift delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/domain/sqlite_vector_store.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/utils/document_uuid.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/utils/hash.dart delete mode 100644 app/lib/features/geckoview/features/tabs/features/vector_store/utils/markdown_document_splitter.dart delete mode 100644 app/lib/utils/langchain_utils.dart diff --git a/app/lib/core/providers/models.dart b/app/lib/core/providers/models.dart deleted file mode 100644 index c5d37d6f..00000000 --- a/app/lib/core/providers/models.dart +++ /dev/null @@ -1,40 +0,0 @@ -import 'package:langchain_openai/langchain_openai.dart'; -import 'package:riverpod/riverpod.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; - -part 'models.g.dart'; - -// @Riverpod(keepAlive: true) -// ChatOpenAI summarizerModel(Ref ref) { -// final openaiApiKey = Platform.environment['OPENAI_API_KEY']; -// return ChatOpenAI( -// apiKey: openaiApiKey, -// baseUrl: 'https://api.together.xyz/v1', -// defaultOptions: const ChatOpenAIOptions(model: 'deepseek-ai/DeepSeek-V3'), -// ); -// } - -@Riverpod(keepAlive: true) -ChatOpenAI chatModel(Ref ref) { - const openaiApiKey = 'FfTBKW1t5dOOXVoiaF2hbXPAHiz3fvfy'; - return ChatOpenAI( - apiKey: openaiApiKey, - baseUrl: 'https://api.deepinfra.com/v1/openai', - defaultOptions: const ChatOpenAIOptions( - model: 'meta-llama/Llama-3.3-70B-Instruct-Turbo', - ), - ); -} - -@Riverpod(keepAlive: true) -OpenAIEmbeddings embeddingModel(Ref ref) { - const openaiApiKey = 'FfTBKW1t5dOOXVoiaF2hbXPAHiz3fvfy'; - return OpenAIEmbeddings( - apiKey: openaiApiKey, - baseUrl: 'https://api.deepinfra.com/v1/openai', - model: 'BAAI/bge-m3', - ); -} - -@Riverpod(keepAlive: true) -int embeddingDimensions(Ref ref) => 1024; diff --git a/app/lib/core/providers/models.g.dart b/app/lib/core/providers/models.g.dart deleted file mode 100644 index 06f64eab..00000000 --- a/app/lib/core/providers/models.g.dart +++ /dev/null @@ -1,63 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'models.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$chatModelHash() => r'9e44e8fa1d688968841933aac8e13212cf41f4ed'; - -/// See also [chatModel]. -@ProviderFor(chatModel) -final chatModelProvider = Provider.internal( - chatModel, - name: r'chatModelProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') ? null : _$chatModelHash, - dependencies: null, - allTransitiveDependencies: null, -); - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -typedef ChatModelRef = ProviderRef; -String _$embeddingModelHash() => r'40111a5f08b43afe6edd399ff0e6c022b3387265'; - -/// See also [embeddingModel]. -@ProviderFor(embeddingModel) -final embeddingModelProvider = Provider.internal( - embeddingModel, - name: r'embeddingModelProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$embeddingModelHash, - dependencies: null, - allTransitiveDependencies: null, -); - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -typedef EmbeddingModelRef = ProviderRef; -String _$embeddingDimensionsHash() => - r'e34f544a18a7e2bd6a479b21f2016b0c0e2ba96c'; - -/// See also [embeddingDimensions]. -@ProviderFor(embeddingDimensions) -final embeddingDimensionsProvider = Provider.internal( - embeddingDimensions, - name: r'embeddingDimensionsProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$embeddingDimensionsHash, - dependencies: null, - allTransitiveDependencies: null, -); - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -typedef EmbeddingDimensionsRef = ProviderRef; -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/chat/features/chat_store/data/database/converters/chat_metadata_converter.dart b/app/lib/features/chat/features/chat_store/data/database/converters/chat_metadata_converter.dart deleted file mode 100644 index 6e3d0ea6..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/converters/chat_metadata_converter.dart +++ /dev/null @@ -1,18 +0,0 @@ -import 'dart:convert'; - -import 'package:drift/drift.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; - -class ChatMetadataConverter extends TypeConverter { - const ChatMetadataConverter(); - - @override - ChatMetadata fromSql(String fromDb) { - return ChatMetadata.fromJson(jsonDecode(fromDb) as Map); - } - - @override - String toSql(ChatMetadata value) { - return jsonEncode(value.toJson()); - } -} diff --git a/app/lib/features/chat/features/chat_store/data/database/daos/messages.dart b/app/lib/features/chat/features/chat_store/data/database/daos/messages.dart deleted file mode 100644 index 3109104e..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/daos/messages.dart +++ /dev/null @@ -1,137 +0,0 @@ -import 'dart:convert'; - -import 'package:drift/drift.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:lexo_rank/lexo_rank.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/database.dart'; - -part 'messages.g.dart'; - -@DriftAccessor() -class MessagesDao extends DatabaseAccessor - with _$MessagesDaoMixin { - MessagesDao(super.db); - - Selectable getMessageCount({required String chatId}) { - return db.chatMessage.count(where: (row) => row.chatId.equals(chatId)); - } - - Selectable getChatMessages({required String chatId}) { - return db.chatMessage.select() - ..where((t) => t.chatId.equals(chatId)) - ..orderBy([(row) => OrderingTerm(expression: row.orderKey)]); - } - - Selectable messageExists(String messageId) { - final existsExpression = existsQuery( - db.chatMessage.select()..where((row) => row.messageId.equals(messageId)), - ); - - return db - .selectExpressions([existsExpression]) - .map((row) => row.read(existsExpression)!); - } - - SingleOrNullSelectable getMessageIndex({ - required String chatId, - required String messageId, - }) { - return db - .messageIndex(chatId: chatId, messageId: messageId) - .map((rowId) => rowId - 1); - } - - Future insertMessage({ - required String chatId, - required Message message, - int? index, - }) async { - if (await messageExists(message.id).getSingle()) { - return null; - } - - return db.transaction(() async { - final String orderKey; - if (index != null) { - if (index <= 0) { - orderKey = - await db.leadingOrderKey(bucket: 0, chatId: chatId).getSingle(); - } else { - orderKey = - await db - .orderKeyBeforeIndex(chatId: chatId, rowIdx: index + 1) - .getSingle(); - } - } else { - orderKey = - await db.trailingOrderKey(bucket: 0, chatId: chatId).getSingle(); - } - - await db.chatMessage.insertOne( - ChatMessageCompanion.insert( - messageId: message.id, - chatId: chatId, - orderKey: orderKey, - messageJson: jsonEncode(message.toJson()), - ), - ); - - final rowIdx = - await db - .messageIndex(chatId: chatId, messageId: message.id) - .getSingle(); - - return rowIdx - 1; - }); - } - - Future removeMessage({required String messageId}) { - return db.chatMessage.deleteWhere((row) => row.messageId.equals(messageId)); - } - - Future updateMessage({ - required String messageId, - required Message message, - }) { - return (db.chatMessage.update() - ..where((row) => row.messageId.equals(messageId))) - .write( - ChatMessageCompanion( - messageJson: Value(jsonEncode(message.toJson())), - ), - ); - } - - Future deleteChatMessages({required String chatId}) { - return (db.chatMessage.delete()..where((t) => t.chatId.equals(chatId))) - .go(); - } - - Future replaceChatMessages({ - required String chatId, - required List messages, - }) { - return db.transaction(() async { - await deleteChatMessages(chatId: chatId); - - var currentOrderKey = - await db.leadingOrderKey(bucket: 0, chatId: chatId).getSingle(); - - await db.chatMessage.insertAll( - messages.map((message) { - final insertable = ChatMessageCompanion.insert( - messageId: message.id, - chatId: chatId, - messageJson: jsonEncode(message.toJson()), - orderKey: currentOrderKey, - // timestamp: DateTime.now(), - ); - - currentOrderKey = LexoRank.parse(currentOrderKey).genPrev().value; - - return insertable; - }), - ); - }); - } -} diff --git a/app/lib/features/chat/features/chat_store/data/database/daos/messages.g.dart b/app/lib/features/chat/features/chat_store/data/database/daos/messages.g.dart deleted file mode 100644 index 8c7718c5..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/daos/messages.g.dart +++ /dev/null @@ -1,6 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'messages.dart'; - -// ignore_for_file: type=lint -mixin _$MessagesDaoMixin on DatabaseAccessor {} diff --git a/app/lib/features/chat/features/chat_store/data/database/daos/metadata.dart b/app/lib/features/chat/features/chat_store/data/database/daos/metadata.dart deleted file mode 100644 index 049cb5d2..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/daos/metadata.dart +++ /dev/null @@ -1,27 +0,0 @@ -import 'package:drift/drift.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/database.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; - -part 'metadata.g.dart'; - -@DriftAccessor() -class MetadataDao extends DatabaseAccessor - with _$MetadataDaoMixin { - MetadataDao(super.db); - - SingleOrNullSelectable getChatMetadata(String chatId) { - final statement = - db.chatData.selectOnly() - ..addColumns([db.chatData.metadata]) - ..where(db.chatData.chatId.equals(chatId)); - - return statement.map((row) => row.readWithConverter(db.chatData.metadata)!); - } - - Future updateChatMetadata(String chatId, ChatMetadata metadata) { - return db.chatData.insertOne( - ChatDataCompanion.insert(chatId: chatId, metadata: metadata), - mode: InsertMode.insertOrReplace, - ); - } -} diff --git a/app/lib/features/chat/features/chat_store/data/database/daos/metadata.g.dart b/app/lib/features/chat/features/chat_store/data/database/daos/metadata.g.dart deleted file mode 100644 index c5e5a202..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/daos/metadata.g.dart +++ /dev/null @@ -1,6 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'metadata.dart'; - -// ignore_for_file: type=lint -mixin _$MetadataDaoMixin on DatabaseAccessor {} diff --git a/app/lib/features/chat/features/chat_store/data/database/database.dart b/app/lib/features/chat/features/chat_store/data/database/database.dart deleted file mode 100644 index 15cc76e2..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/database.dart +++ /dev/null @@ -1,15 +0,0 @@ -import 'package:drift/drift.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/converters/chat_metadata_converter.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/daos/messages.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/daos/metadata.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; - -part 'database.g.dart'; - -@DriftDatabase(include: {'database.drift'}, daos: [MessagesDao, MetadataDao]) -class ChatDatabase extends _$ChatDatabase { - ChatDatabase(super.e); - - @override - int get schemaVersion => 1; -} diff --git a/app/lib/features/chat/features/chat_store/data/database/database.drift b/app/lib/features/chat/features/chat_store/data/database/database.drift deleted file mode 100644 index 120b571e..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/database.drift +++ /dev/null @@ -1,65 +0,0 @@ -import 'package:weblibre/features/chat/features/chat_store/data/database/converters/chat_metadata_converter.dart'; - -CREATE TABLE chat_data ( - chat_id TEXT PRIMARY KEY NOT NULL, - metadata TEXT NOT NULL MAPPED BY `const ChatMetadataConverter()` -); - -CREATE TABLE chat_message ( - message_id TEXT PRIMARY KEY NOT NULL, - chat_id TEXT NOT NULL, - order_key TEXT NOT NULL, - message_json TEXT NOT NULL -); -CREATE UNIQUE INDEX group_order_idx on chat_message (chat_id, order_key); - -leadingOrderKey: - SELECT lexo_rank_previous( - :bucket, - ( - SELECT order_key - FROM chat_message - WHERE chat_id = :chat_id - ORDER BY order_key - LIMIT 1 - ) - ); - -trailingOrderKey: - SELECT lexo_rank_next( - :bucket, - ( - SELECT order_key - FROM chat_message - WHERE chat_id = :chat_id - ORDER BY order_key DESC - LIMIT 1 - ) - ); - -orderKeyBeforeIndex: - WITH ordered_table AS - ( - SELECT message_id, - order_key, - LEAD(order_key) OVER (ORDER BY order_key) AS next_order_key, - ROW_NUMBER() OVER (ORDER BY order_key) AS row_idx, - COUNT(*) OVER () as total_rows - FROM chat_message - WHERE chat_id = :chat_id - ) - SELECT lexo_rank_reorder_after(order_key, next_order_key) - FROM ordered_table - WHERE row_idx = MIN(:row_idx, total_rows); - -messageIndex: - WITH ordered_table AS - ( - SELECT message_id, - ROW_NUMBER() OVER (ORDER BY order_key) AS row_idx - FROM chat_message - WHERE chat_id = :chat_id - ) - SELECT row_idx - FROM ordered_table - WHERE message_id = :message_id; \ No newline at end of file diff --git a/app/lib/features/chat/features/chat_store/data/database/database.g.dart b/app/lib/features/chat/features/chat_store/data/database/database.g.dart deleted file mode 100644 index c50b6d5e..00000000 --- a/app/lib/features/chat/features/chat_store/data/database/database.g.dart +++ /dev/null @@ -1,871 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'database.dart'; - -// ignore_for_file: type=lint -class ChatData extends Table with TableInfo { - @override - final GeneratedDatabase attachedDatabase; - final String? _alias; - ChatData(this.attachedDatabase, [this._alias]); - late final GeneratedColumn chatId = GeneratedColumn( - 'chat_id', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'PRIMARY KEY NOT NULL', - ); - late final GeneratedColumnWithTypeConverter metadata = - GeneratedColumn( - 'metadata', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ).withConverter(ChatData.$convertermetadata); - @override - List get $columns => [chatId, metadata]; - @override - String get aliasedName => _alias ?? actualTableName; - @override - String get actualTableName => $name; - static const String $name = 'chat_data'; - @override - Set get $primaryKey => {chatId}; - @override - ChatDataData map(Map data, {String? tablePrefix}) { - final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; - return ChatDataData( - chatId: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}chat_id'], - )!, - metadata: ChatData.$convertermetadata.fromSql( - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}metadata'], - )!, - ), - ); - } - - @override - ChatData createAlias(String alias) { - return ChatData(attachedDatabase, alias); - } - - static TypeConverter $convertermetadata = - const ChatMetadataConverter(); - @override - bool get dontWriteConstraints => true; -} - -class ChatDataData extends DataClass implements Insertable { - final String chatId; - final ChatMetadata metadata; - const ChatDataData({required this.chatId, required this.metadata}); - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - map['chat_id'] = Variable(chatId); - { - map['metadata'] = Variable( - ChatData.$convertermetadata.toSql(metadata), - ); - } - return map; - } - - factory ChatDataData.fromJson( - Map json, { - ValueSerializer? serializer, - }) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return ChatDataData( - chatId: serializer.fromJson(json['chat_id']), - metadata: serializer.fromJson(json['metadata']), - ); - } - @override - Map toJson({ValueSerializer? serializer}) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return { - 'chat_id': serializer.toJson(chatId), - 'metadata': serializer.toJson(metadata), - }; - } - - ChatDataData copyWith({String? chatId, ChatMetadata? metadata}) => - ChatDataData( - chatId: chatId ?? this.chatId, - metadata: metadata ?? this.metadata, - ); - ChatDataData copyWithCompanion(ChatDataCompanion data) { - return ChatDataData( - chatId: data.chatId.present ? data.chatId.value : this.chatId, - metadata: data.metadata.present ? data.metadata.value : this.metadata, - ); - } - - @override - String toString() { - return (StringBuffer('ChatDataData(') - ..write('chatId: $chatId, ') - ..write('metadata: $metadata') - ..write(')')) - .toString(); - } - - @override - int get hashCode => Object.hash(chatId, metadata); - @override - bool operator ==(Object other) => - identical(this, other) || - (other is ChatDataData && - other.chatId == this.chatId && - other.metadata == this.metadata); -} - -class ChatDataCompanion extends UpdateCompanion { - final Value chatId; - final Value metadata; - final Value rowid; - const ChatDataCompanion({ - this.chatId = const Value.absent(), - this.metadata = const Value.absent(), - this.rowid = const Value.absent(), - }); - ChatDataCompanion.insert({ - required String chatId, - required ChatMetadata metadata, - this.rowid = const Value.absent(), - }) : chatId = Value(chatId), - metadata = Value(metadata); - static Insertable custom({ - Expression? chatId, - Expression? metadata, - Expression? rowid, - }) { - return RawValuesInsertable({ - if (chatId != null) 'chat_id': chatId, - if (metadata != null) 'metadata': metadata, - if (rowid != null) 'rowid': rowid, - }); - } - - ChatDataCompanion copyWith({ - Value? chatId, - Value? metadata, - Value? rowid, - }) { - return ChatDataCompanion( - chatId: chatId ?? this.chatId, - metadata: metadata ?? this.metadata, - rowid: rowid ?? this.rowid, - ); - } - - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - if (chatId.present) { - map['chat_id'] = Variable(chatId.value); - } - if (metadata.present) { - map['metadata'] = Variable( - ChatData.$convertermetadata.toSql(metadata.value), - ); - } - if (rowid.present) { - map['rowid'] = Variable(rowid.value); - } - return map; - } - - @override - String toString() { - return (StringBuffer('ChatDataCompanion(') - ..write('chatId: $chatId, ') - ..write('metadata: $metadata, ') - ..write('rowid: $rowid') - ..write(')')) - .toString(); - } -} - -class ChatMessage extends Table with TableInfo { - @override - final GeneratedDatabase attachedDatabase; - final String? _alias; - ChatMessage(this.attachedDatabase, [this._alias]); - late final GeneratedColumn messageId = GeneratedColumn( - 'message_id', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'PRIMARY KEY NOT NULL', - ); - late final GeneratedColumn chatId = GeneratedColumn( - 'chat_id', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - late final GeneratedColumn orderKey = GeneratedColumn( - 'order_key', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - late final GeneratedColumn messageJson = GeneratedColumn( - 'message_json', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - @override - List get $columns => [ - messageId, - chatId, - orderKey, - messageJson, - ]; - @override - String get aliasedName => _alias ?? actualTableName; - @override - String get actualTableName => $name; - static const String $name = 'chat_message'; - @override - Set get $primaryKey => {messageId}; - @override - ChatMessageData map(Map data, {String? tablePrefix}) { - final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; - return ChatMessageData( - messageId: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}message_id'], - )!, - chatId: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}chat_id'], - )!, - orderKey: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}order_key'], - )!, - messageJson: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}message_json'], - )!, - ); - } - - @override - ChatMessage createAlias(String alias) { - return ChatMessage(attachedDatabase, alias); - } - - @override - bool get dontWriteConstraints => true; -} - -class ChatMessageData extends DataClass implements Insertable { - final String messageId; - final String chatId; - final String orderKey; - final String messageJson; - const ChatMessageData({ - required this.messageId, - required this.chatId, - required this.orderKey, - required this.messageJson, - }); - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - map['message_id'] = Variable(messageId); - map['chat_id'] = Variable(chatId); - map['order_key'] = Variable(orderKey); - map['message_json'] = Variable(messageJson); - return map; - } - - factory ChatMessageData.fromJson( - Map json, { - ValueSerializer? serializer, - }) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return ChatMessageData( - messageId: serializer.fromJson(json['message_id']), - chatId: serializer.fromJson(json['chat_id']), - orderKey: serializer.fromJson(json['order_key']), - messageJson: serializer.fromJson(json['message_json']), - ); - } - @override - Map toJson({ValueSerializer? serializer}) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return { - 'message_id': serializer.toJson(messageId), - 'chat_id': serializer.toJson(chatId), - 'order_key': serializer.toJson(orderKey), - 'message_json': serializer.toJson(messageJson), - }; - } - - ChatMessageData copyWith({ - String? messageId, - String? chatId, - String? orderKey, - String? messageJson, - }) => ChatMessageData( - messageId: messageId ?? this.messageId, - chatId: chatId ?? this.chatId, - orderKey: orderKey ?? this.orderKey, - messageJson: messageJson ?? this.messageJson, - ); - ChatMessageData copyWithCompanion(ChatMessageCompanion data) { - return ChatMessageData( - messageId: data.messageId.present ? data.messageId.value : this.messageId, - chatId: data.chatId.present ? data.chatId.value : this.chatId, - orderKey: data.orderKey.present ? data.orderKey.value : this.orderKey, - messageJson: - data.messageJson.present ? data.messageJson.value : this.messageJson, - ); - } - - @override - String toString() { - return (StringBuffer('ChatMessageData(') - ..write('messageId: $messageId, ') - ..write('chatId: $chatId, ') - ..write('orderKey: $orderKey, ') - ..write('messageJson: $messageJson') - ..write(')')) - .toString(); - } - - @override - int get hashCode => Object.hash(messageId, chatId, orderKey, messageJson); - @override - bool operator ==(Object other) => - identical(this, other) || - (other is ChatMessageData && - other.messageId == this.messageId && - other.chatId == this.chatId && - other.orderKey == this.orderKey && - other.messageJson == this.messageJson); -} - -class ChatMessageCompanion extends UpdateCompanion { - final Value messageId; - final Value chatId; - final Value orderKey; - final Value messageJson; - final Value rowid; - const ChatMessageCompanion({ - this.messageId = const Value.absent(), - this.chatId = const Value.absent(), - this.orderKey = const Value.absent(), - this.messageJson = const Value.absent(), - this.rowid = const Value.absent(), - }); - ChatMessageCompanion.insert({ - required String messageId, - required String chatId, - required String orderKey, - required String messageJson, - this.rowid = const Value.absent(), - }) : messageId = Value(messageId), - chatId = Value(chatId), - orderKey = Value(orderKey), - messageJson = Value(messageJson); - static Insertable custom({ - Expression? messageId, - Expression? chatId, - Expression? orderKey, - Expression? messageJson, - Expression? rowid, - }) { - return RawValuesInsertable({ - if (messageId != null) 'message_id': messageId, - if (chatId != null) 'chat_id': chatId, - if (orderKey != null) 'order_key': orderKey, - if (messageJson != null) 'message_json': messageJson, - if (rowid != null) 'rowid': rowid, - }); - } - - ChatMessageCompanion copyWith({ - Value? messageId, - Value? chatId, - Value? orderKey, - Value? messageJson, - Value? rowid, - }) { - return ChatMessageCompanion( - messageId: messageId ?? this.messageId, - chatId: chatId ?? this.chatId, - orderKey: orderKey ?? this.orderKey, - messageJson: messageJson ?? this.messageJson, - rowid: rowid ?? this.rowid, - ); - } - - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - if (messageId.present) { - map['message_id'] = Variable(messageId.value); - } - if (chatId.present) { - map['chat_id'] = Variable(chatId.value); - } - if (orderKey.present) { - map['order_key'] = Variable(orderKey.value); - } - if (messageJson.present) { - map['message_json'] = Variable(messageJson.value); - } - if (rowid.present) { - map['rowid'] = Variable(rowid.value); - } - return map; - } - - @override - String toString() { - return (StringBuffer('ChatMessageCompanion(') - ..write('messageId: $messageId, ') - ..write('chatId: $chatId, ') - ..write('orderKey: $orderKey, ') - ..write('messageJson: $messageJson, ') - ..write('rowid: $rowid') - ..write(')')) - .toString(); - } -} - -abstract class _$ChatDatabase extends GeneratedDatabase { - _$ChatDatabase(QueryExecutor e) : super(e); - $ChatDatabaseManager get managers => $ChatDatabaseManager(this); - late final ChatData chatData = ChatData(this); - late final ChatMessage chatMessage = ChatMessage(this); - late final Index groupOrderIdx = Index( - 'group_order_idx', - 'CREATE UNIQUE INDEX group_order_idx ON chat_message (chat_id, order_key)', - ); - late final MessagesDao messagesDao = MessagesDao(this as ChatDatabase); - late final MetadataDao metadataDao = MetadataDao(this as ChatDatabase); - Selectable leadingOrderKey({ - required int bucket, - required String chatId, - }) { - return customSelect( - 'SELECT lexo_rank_previous(?1, (SELECT order_key FROM chat_message WHERE chat_id = ?2 ORDER BY order_key LIMIT 1)) AS _c0', - variables: [Variable(bucket), Variable(chatId)], - readsFrom: {chatMessage}, - ).map((QueryRow row) => row.read('_c0')); - } - - Selectable trailingOrderKey({ - required int bucket, - required String chatId, - }) { - return customSelect( - 'SELECT lexo_rank_next(?1, (SELECT order_key FROM chat_message WHERE chat_id = ?2 ORDER BY order_key DESC LIMIT 1)) AS _c0', - variables: [Variable(bucket), Variable(chatId)], - readsFrom: {chatMessage}, - ).map((QueryRow row) => row.read('_c0')); - } - - Selectable orderKeyBeforeIndex({ - required String chatId, - required int rowIdx, - }) { - return customSelect( - 'WITH ordered_table AS (SELECT message_id, order_key, LEAD(order_key)OVER (ORDER BY order_key RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS next_order_key, ROW_NUMBER()OVER (ORDER BY order_key RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS row_idx, COUNT(*)OVER (RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS total_rows FROM chat_message WHERE chat_id = ?1) SELECT lexo_rank_reorder_after(order_key, next_order_key) AS _c0 FROM ordered_table WHERE row_idx = MIN(?2, total_rows)', - variables: [Variable(chatId), Variable(rowIdx)], - readsFrom: {chatMessage}, - ).map((QueryRow row) => row.read('_c0')); - } - - Selectable messageIndex({ - required String chatId, - required String messageId, - }) { - return customSelect( - 'WITH ordered_table AS (SELECT message_id, ROW_NUMBER()OVER (ORDER BY order_key RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW EXCLUDE NO OTHERS) AS row_idx FROM chat_message WHERE chat_id = ?1) SELECT row_idx FROM ordered_table WHERE message_id = ?2', - variables: [Variable(chatId), Variable(messageId)], - readsFrom: {chatMessage}, - ).map((QueryRow row) => row.read('row_idx')); - } - - @override - Iterable> get allTables => - allSchemaEntities.whereType>(); - @override - List get allSchemaEntities => [ - chatData, - chatMessage, - groupOrderIdx, - ]; -} - -typedef $ChatDataCreateCompanionBuilder = - ChatDataCompanion Function({ - required String chatId, - required ChatMetadata metadata, - Value rowid, - }); -typedef $ChatDataUpdateCompanionBuilder = - ChatDataCompanion Function({ - Value chatId, - Value metadata, - Value rowid, - }); - -class $ChatDataFilterComposer extends Composer<_$ChatDatabase, ChatData> { - $ChatDataFilterComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnFilters get chatId => $composableBuilder( - column: $table.chatId, - builder: (column) => ColumnFilters(column), - ); - - ColumnWithTypeConverterFilters - get metadata => $composableBuilder( - column: $table.metadata, - builder: (column) => ColumnWithTypeConverterFilters(column), - ); -} - -class $ChatDataOrderingComposer extends Composer<_$ChatDatabase, ChatData> { - $ChatDataOrderingComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnOrderings get chatId => $composableBuilder( - column: $table.chatId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get metadata => $composableBuilder( - column: $table.metadata, - builder: (column) => ColumnOrderings(column), - ); -} - -class $ChatDataAnnotationComposer extends Composer<_$ChatDatabase, ChatData> { - $ChatDataAnnotationComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - GeneratedColumn get chatId => - $composableBuilder(column: $table.chatId, builder: (column) => column); - - GeneratedColumnWithTypeConverter get metadata => - $composableBuilder(column: $table.metadata, builder: (column) => column); -} - -class $ChatDataTableManager - extends - RootTableManager< - _$ChatDatabase, - ChatData, - ChatDataData, - $ChatDataFilterComposer, - $ChatDataOrderingComposer, - $ChatDataAnnotationComposer, - $ChatDataCreateCompanionBuilder, - $ChatDataUpdateCompanionBuilder, - ( - ChatDataData, - BaseReferences<_$ChatDatabase, ChatData, ChatDataData>, - ), - ChatDataData, - PrefetchHooks Function() - > { - $ChatDataTableManager(_$ChatDatabase db, ChatData table) - : super( - TableManagerState( - db: db, - table: table, - createFilteringComposer: - () => $ChatDataFilterComposer($db: db, $table: table), - createOrderingComposer: - () => $ChatDataOrderingComposer($db: db, $table: table), - createComputedFieldComposer: - () => $ChatDataAnnotationComposer($db: db, $table: table), - updateCompanionCallback: - ({ - Value chatId = const Value.absent(), - Value metadata = const Value.absent(), - Value rowid = const Value.absent(), - }) => ChatDataCompanion( - chatId: chatId, - metadata: metadata, - rowid: rowid, - ), - createCompanionCallback: - ({ - required String chatId, - required ChatMetadata metadata, - Value rowid = const Value.absent(), - }) => ChatDataCompanion.insert( - chatId: chatId, - metadata: metadata, - rowid: rowid, - ), - withReferenceMapper: - (p0) => - p0 - .map( - (e) => ( - e.readTable(table), - BaseReferences(db, table, e), - ), - ) - .toList(), - prefetchHooksCallback: null, - ), - ); -} - -typedef $ChatDataProcessedTableManager = - ProcessedTableManager< - _$ChatDatabase, - ChatData, - ChatDataData, - $ChatDataFilterComposer, - $ChatDataOrderingComposer, - $ChatDataAnnotationComposer, - $ChatDataCreateCompanionBuilder, - $ChatDataUpdateCompanionBuilder, - (ChatDataData, BaseReferences<_$ChatDatabase, ChatData, ChatDataData>), - ChatDataData, - PrefetchHooks Function() - >; -typedef $ChatMessageCreateCompanionBuilder = - ChatMessageCompanion Function({ - required String messageId, - required String chatId, - required String orderKey, - required String messageJson, - Value rowid, - }); -typedef $ChatMessageUpdateCompanionBuilder = - ChatMessageCompanion Function({ - Value messageId, - Value chatId, - Value orderKey, - Value messageJson, - Value rowid, - }); - -class $ChatMessageFilterComposer extends Composer<_$ChatDatabase, ChatMessage> { - $ChatMessageFilterComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnFilters get messageId => $composableBuilder( - column: $table.messageId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get chatId => $composableBuilder( - column: $table.chatId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get orderKey => $composableBuilder( - column: $table.orderKey, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get messageJson => $composableBuilder( - column: $table.messageJson, - builder: (column) => ColumnFilters(column), - ); -} - -class $ChatMessageOrderingComposer - extends Composer<_$ChatDatabase, ChatMessage> { - $ChatMessageOrderingComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnOrderings get messageId => $composableBuilder( - column: $table.messageId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get chatId => $composableBuilder( - column: $table.chatId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get orderKey => $composableBuilder( - column: $table.orderKey, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get messageJson => $composableBuilder( - column: $table.messageJson, - builder: (column) => ColumnOrderings(column), - ); -} - -class $ChatMessageAnnotationComposer - extends Composer<_$ChatDatabase, ChatMessage> { - $ChatMessageAnnotationComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - GeneratedColumn get messageId => - $composableBuilder(column: $table.messageId, builder: (column) => column); - - GeneratedColumn get chatId => - $composableBuilder(column: $table.chatId, builder: (column) => column); - - GeneratedColumn get orderKey => - $composableBuilder(column: $table.orderKey, builder: (column) => column); - - GeneratedColumn get messageJson => $composableBuilder( - column: $table.messageJson, - builder: (column) => column, - ); -} - -class $ChatMessageTableManager - extends - RootTableManager< - _$ChatDatabase, - ChatMessage, - ChatMessageData, - $ChatMessageFilterComposer, - $ChatMessageOrderingComposer, - $ChatMessageAnnotationComposer, - $ChatMessageCreateCompanionBuilder, - $ChatMessageUpdateCompanionBuilder, - ( - ChatMessageData, - BaseReferences<_$ChatDatabase, ChatMessage, ChatMessageData>, - ), - ChatMessageData, - PrefetchHooks Function() - > { - $ChatMessageTableManager(_$ChatDatabase db, ChatMessage table) - : super( - TableManagerState( - db: db, - table: table, - createFilteringComposer: - () => $ChatMessageFilterComposer($db: db, $table: table), - createOrderingComposer: - () => $ChatMessageOrderingComposer($db: db, $table: table), - createComputedFieldComposer: - () => $ChatMessageAnnotationComposer($db: db, $table: table), - updateCompanionCallback: - ({ - Value messageId = const Value.absent(), - Value chatId = const Value.absent(), - Value orderKey = const Value.absent(), - Value messageJson = const Value.absent(), - Value rowid = const Value.absent(), - }) => ChatMessageCompanion( - messageId: messageId, - chatId: chatId, - orderKey: orderKey, - messageJson: messageJson, - rowid: rowid, - ), - createCompanionCallback: - ({ - required String messageId, - required String chatId, - required String orderKey, - required String messageJson, - Value rowid = const Value.absent(), - }) => ChatMessageCompanion.insert( - messageId: messageId, - chatId: chatId, - orderKey: orderKey, - messageJson: messageJson, - rowid: rowid, - ), - withReferenceMapper: - (p0) => - p0 - .map( - (e) => ( - e.readTable(table), - BaseReferences(db, table, e), - ), - ) - .toList(), - prefetchHooksCallback: null, - ), - ); -} - -typedef $ChatMessageProcessedTableManager = - ProcessedTableManager< - _$ChatDatabase, - ChatMessage, - ChatMessageData, - $ChatMessageFilterComposer, - $ChatMessageOrderingComposer, - $ChatMessageAnnotationComposer, - $ChatMessageCreateCompanionBuilder, - $ChatMessageUpdateCompanionBuilder, - ( - ChatMessageData, - BaseReferences<_$ChatDatabase, ChatMessage, ChatMessageData>, - ), - ChatMessageData, - PrefetchHooks Function() - >; - -class $ChatDatabaseManager { - final _$ChatDatabase _db; - $ChatDatabaseManager(this._db); - $ChatDataTableManager get chatData => - $ChatDataTableManager(_db, _db.chatData); - $ChatMessageTableManager get chatMessage => - $ChatMessageTableManager(_db, _db.chatMessage); -} diff --git a/app/lib/features/chat/features/chat_store/data/models/chat_metadata.dart b/app/lib/features/chat/features/chat_store/data/models/chat_metadata.dart deleted file mode 100644 index e0d44ca5..00000000 --- a/app/lib/features/chat/features/chat_store/data/models/chat_metadata.dart +++ /dev/null @@ -1,20 +0,0 @@ -import 'package:fast_equatable/fast_equatable.dart'; -import 'package:json_annotation/json_annotation.dart'; - -part 'chat_metadata.g.dart'; - -@JsonSerializable() -class ChatMetadata with FastEquatable { - final String? mainDocumentId; - final String? contextId; - - ChatMetadata({this.mainDocumentId, this.contextId}); - - factory ChatMetadata.fromJson(Map json) => - _$ChatMetadataFromJson(json); - - Map toJson() => _$ChatMetadataToJson(this); - - @override - List get hashParameters => [mainDocumentId, contextId]; -} diff --git a/app/lib/features/chat/features/chat_store/data/models/chat_metadata.g.dart b/app/lib/features/chat/features/chat_store/data/models/chat_metadata.g.dart deleted file mode 100644 index 526b847c..00000000 --- a/app/lib/features/chat/features/chat_store/data/models/chat_metadata.g.dart +++ /dev/null @@ -1,18 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'chat_metadata.dart'; - -// ************************************************************************** -// JsonSerializableGenerator -// ************************************************************************** - -ChatMetadata _$ChatMetadataFromJson(Map json) => ChatMetadata( - mainDocumentId: json['mainDocumentId'] as String?, - contextId: json['contextId'] as String?, -); - -Map _$ChatMetadataToJson(ChatMetadata instance) => - { - 'mainDocumentId': instance.mainDocumentId, - 'contextId': instance.contextId, - }; diff --git a/app/lib/features/chat/features/chat_store/data/models/message_types.dart b/app/lib/features/chat/features/chat_store/data/models/message_types.dart deleted file mode 100644 index 789fe71c..00000000 --- a/app/lib/features/chat/features/chat_store/data/models/message_types.dart +++ /dev/null @@ -1,22 +0,0 @@ -import 'package:flutter_chat_core/flutter_chat_core.dart'; - -enum MessageAuthor { - ///langchain SystemChatMessage - system(User(id: 'system')), - - ///langchain HumanChatMessage - human(User(id: 'human')), - - ///langchain AIChatMessage - ai(User(id: 'ai')), - - ///langchain ToolChatMessage - tool(User(id: 'tool')), - - ///langchain CustomChatMessage - custom(User(id: 'custom')); - - final User user; - - const MessageAuthor(this.user); -} diff --git a/app/lib/features/chat/features/chat_store/data/providers.dart b/app/lib/features/chat/features/chat_store/data/providers.dart deleted file mode 100644 index e4b0214c..00000000 --- a/app/lib/features/chat/features/chat_store/data/providers.dart +++ /dev/null @@ -1,50 +0,0 @@ -import 'package:drift/drift.dart'; -import 'package:drift/native.dart'; -import 'package:path/path.dart' as p; -import 'package:path_provider/path_provider.dart' as path_provider; -import 'package:riverpod/riverpod.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:sqlite3/sqlite3.dart'; -import 'package:sqlite3_flutter_libs/sqlite3_flutter_libs.dart'; -import 'package:universal_io/io.dart'; -import 'package:weblibre/data/database/functions/lexo_rank_functions.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/database.dart'; - -part 'providers.g.dart'; - -@Riverpod(keepAlive: true) -ChatDatabase chatDatabase(Ref ref) { - final db = ChatDatabase( - LazyDatabase(() async { - // put the database file, called db.sqlite here, into the documents folder - // for your app. - final dbFolder = await path_provider.getApplicationDocumentsDirectory(); - final file = File(p.join(dbFolder.path, 'chat.db')); - - // Also work around limitations on old Android versions - if (Platform.isAndroid) { - await applyWorkaroundToOpenSqlite3OnOldAndroidVersions(); - } - - // Make sqlite3 pick a more suitable location for temporary files - the - // one from the system may be inaccessible due to sandboxing. - final cachebase = (await path_provider.getTemporaryDirectory()).path; - // We can't access /tmp on Android, which sqlite3 would try by default. - // Explicitly tell it about the correct temporary directory. - sqlite3.tempDirectory = cachebase; - - return NativeDatabase.createInBackground( - file, - setup: (database) { - registerLexorankFunctions(database); - }, - ); - }), - ); - - ref.onDispose(() async { - await db.close(); - }); - - return db; -} diff --git a/app/lib/features/chat/features/chat_store/data/providers.g.dart b/app/lib/features/chat/features/chat_store/data/providers.g.dart deleted file mode 100644 index c5afff20..00000000 --- a/app/lib/features/chat/features/chat_store/data/providers.g.dart +++ /dev/null @@ -1,26 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'providers.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$chatDatabaseHash() => r'd07fef426c6984e6fb2eab9651a5c51aaca98961'; - -/// See also [chatDatabase]. -@ProviderFor(chatDatabase) -final chatDatabaseProvider = Provider.internal( - chatDatabase, - name: r'chatDatabaseProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') ? null : _$chatDatabaseHash, - dependencies: null, - allTransitiveDependencies: null, -); - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -typedef ChatDatabaseRef = ProviderRef; -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/chat/features/chat_store/domain/chat_memory.dart b/app/lib/features/chat/features/chat_store/domain/chat_memory.dart deleted file mode 100644 index f1a1e3c7..00000000 --- a/app/lib/features/chat/features/chat_store/domain/chat_memory.dart +++ /dev/null @@ -1,155 +0,0 @@ -// ignore_for_file: avoid_dynamic_calls we know what we do - -import 'dart:convert'; - -import 'package:collection/collection.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:langchain/langchain.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/message_types.dart'; - -class ChatMemory implements BaseMemory { - final ChatController _controller; - - /// Max number of tokens to use. - final int maxTokenLimit; - - /// Language model to use for counting tokens. - final BaseLanguageModel llm; - - /// The memory key to use for the chat history. - /// This will be passed as input variable to the prompt. - final String memoryKey; - - /// If true, when [loadMemoryVariables] is called, it will return - /// [ChatMessage] objects. If false, it will return a String representation - /// of the messages. - /// - /// Set this to true when you are using a Chat model like `ChatOpenAI`. - /// Set this to false when you are use a text LLM like `OpenAI`. - final bool returnMessages; - - /// The prefix to use for system messages if [returnMessages] is false. - final String systemPrefix; - - /// The prefix to use for human messages if [returnMessages] is false. - final String humanPrefix; - - /// The prefix to use for AI messages if [returnMessages] is false. - final String aiPrefix; - - /// The prefix to use for tool messages if [returnMessages] is false. - final String toolPrefix; - - ChatMemory( - this._controller, { - required this.returnMessages, - this.maxTokenLimit = 2000, - required this.llm, - this.memoryKey = BaseMemory.defaultMemoryKey, - this.systemPrefix = SystemChatMessage.defaultPrefix, - this.humanPrefix = HumanChatMessage.defaultPrefix, - this.aiPrefix = AIChatMessage.defaultPrefix, - this.toolPrefix = ToolChatMessage.defaultPrefix, - }); - - @override - Set get memoryKeys => {memoryKey}; - - @override - Future loadMemoryVariables([ - MemoryInputValues values = const {}, - ]) async { - final messages = - _controller.messages - .whereNot( - (message) => - message.metadata?['hideFromModelChatHistory'] == true, - ) - .map( - (message) => switch (message) { - TextMessage( - authorId: final authorId, - text: final text, - metadata: final metadata, - ) => - () { - if (authorId == MessageAuthor.system.user.id) { - return SystemChatMessage(content: text); - } - if (authorId == MessageAuthor.human.user.id) { - return HumanChatMessage( - content: ChatMessageContent.text(text), - ); - } - if (authorId == MessageAuthor.ai.user.id) { - final toolsRaw = metadata?['toolCalls']; - if (toolsRaw is String) { - return AIChatMessage( - content: text, - toolCalls: _deserializeToolCalls(toolsRaw), - ); - } - return AIChatMessage(content: text); - } - throw UnimplementedError(); - }(), - ImageMessage() => throw UnimplementedError(), - CustomMessage() => throw UnimplementedError(), - UnsupportedMessage() => throw UnimplementedError(), - }, - ) - .toList(); - - int currentBufferLength = await llm.countTokens(PromptValue.chat(messages)); - // Prune buffer if it exceeds max token limit - if (currentBufferLength > maxTokenLimit) { - while (currentBufferLength > maxTokenLimit) { - //remove oldest entry - messages.removeAt(0); - currentBufferLength = await llm.countTokens(PromptValue.chat(messages)); - } - } - - if (returnMessages) { - return {memoryKey: messages}; - } - - return { - memoryKey: messages.toBufferString( - systemPrefix: systemPrefix, - humanPrefix: humanPrefix, - aiPrefix: aiPrefix, - toolPrefix: toolPrefix, - ), - }; - } - - List _deserializeToolCalls(String toolsRaw) { - final decoded = jsonDecode(toolsRaw) as List; - final tools = - decoded - .map( - (tool) => AIChatMessageToolCall( - id: tool['id'] as String, - name: tool['name'] as String, - arguments: tool['arguments'] as Map, - argumentsRaw: tool['argumentsRaw'] as String, - ), - ) - .toList(); - return tools; - } - - @override - Future saveContext({ - required MemoryInputValues inputValues, - required MemoryOutputValues outputValues, - }) { - throw UnimplementedError('This is a read only memory'); - } - - @override - Future clear() { - throw UnimplementedError('This is a read only memory'); - } -} diff --git a/app/lib/features/chat/features/chat_store/domain/controllers/drift_chat_controller.dart b/app/lib/features/chat/features/chat_store/domain/controllers/drift_chat_controller.dart deleted file mode 100644 index bf19a270..00000000 --- a/app/lib/features/chat/features/chat_store/domain/controllers/drift_chat_controller.dart +++ /dev/null @@ -1,166 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; - -import 'package:collection/collection.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:rxdart/rxdart.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/database/daos/messages.dart'; - -class DriftChatController implements ChatController { - final MessagesDao _dao; - final String chatId; - - final _operationsController = StreamController.broadcast(); - - late final ValueStream> _mergedMessagesStream; - late final StreamSubscription> _mergedMessagesStreamSub; - late final BehaviorSubject> _intermediateMessages; - - DriftChatController(this._dao, {required this.chatId}) { - final chatMessagesStream = - _dao - .getChatMessages(chatId: chatId) - .map( - (data) => Message.fromJson( - jsonDecode(data.messageJson) as Map, - ), - ) - .watch(); - - _intermediateMessages = BehaviorSubject.seeded({}); - - _mergedMessagesStream = - Rx.combineLatest2(chatMessagesStream, _intermediateMessages, (a, b) { - if (a.isNotEmpty && b.isNotEmpty) { - return [for (final message in a) b[message.id] ?? message]; - } - - return a; - }).shareValue(); - - //Set initial messages - unawaited( - _mergedMessagesStream.first.then((messages) { - if (messages.isNotEmpty) { - _operationsController.add(ChatOperation.set()); - } - }), - ); - - _mergedMessagesStreamSub = _mergedMessagesStream.listen(null); - } - - @override - List get messages => _mergedMessagesStream.valueOrNull ?? []; - - @override - Future insert(Message message, {int? index}) async { - final insertedIndex = await _dao.insertMessage( - chatId: chatId, - message: message, - index: index, - ); - - if (insertedIndex != null) { - await _mergedMessagesStream - .firstWhere( - (messages) => - messages.length > insertedIndex && - messages[insertedIndex].id == message.id, - ) - .timeout(const Duration(milliseconds: 100)); - - _operationsController.add(ChatOperation.insert(message, insertedIndex)); - } - } - - @override - Future remove(Message message) async { - final index = - await _dao - .getMessageIndex(chatId: chatId, messageId: message.id) - .getSingleOrNull(); - - if (index != null) { - await _dao.removeMessage(messageId: message.id); - await _mergedMessagesStream - .firstWhere( - (messages) => - !messages.any((dbMessage) => dbMessage.id == message.id), - ) - .timeout(const Duration(milliseconds: 100)); - - _operationsController.add(ChatOperation.remove(message, index)); - } - } - - @override - Future update( - Message oldMessage, - Message newMessage, { - bool persistMessage = true, - }) async { - assert(oldMessage.id == newMessage.id); - - if (persistMessage) { - final intermediateMessage = _intermediateMessages.value[newMessage.id]; - - if (intermediateMessage == null && oldMessage == newMessage) { - //nothing to update - return; - } - - final rows = await _dao.updateMessage( - messageId: oldMessage.id, - message: newMessage, - ); - - if (intermediateMessage != null) { - _intermediateMessages.add( - {..._intermediateMessages.value}..remove(newMessage.id), - ); - } - - if (rows <= 0 || intermediateMessage == newMessage) { - //Dont send event - return; - } - } else if (oldMessage != newMessage) { - _intermediateMessages.add({ - ..._intermediateMessages.value, - newMessage.id: newMessage, - }); - } - - await _mergedMessagesStream - .firstWhere( - (messages) => messages.any((message) => message == newMessage), - ) - .timeout(const Duration(milliseconds: 100)); - - _operationsController.add(ChatOperation.update(oldMessage, newMessage)); - } - - @override - Future set(List messages) async { - await _dao.replaceChatMessages(chatId: chatId, messages: messages); - await _mergedMessagesStream - .firstWhere( - (replacedMessages) => - const DeepCollectionEquality().equals(messages, replacedMessages), - ) - .timeout(const Duration(milliseconds: 100)); - - _operationsController.add(ChatOperation.set()); - } - - @override - Stream get operationsStream => _operationsController.stream; - - @override - void dispose() { - unawaited(_intermediateMessages.close()); - unawaited(_mergedMessagesStreamSub.cancel()); - unawaited(_operationsController.close()); - } -} diff --git a/app/lib/features/chat/features/chat_store/domain/providers.dart b/app/lib/features/chat/features/chat_store/domain/providers.dart deleted file mode 100644 index ba637d31..00000000 --- a/app/lib/features/chat/features/chat_store/domain/providers.dart +++ /dev/null @@ -1,25 +0,0 @@ -import 'package:riverpod/riverpod.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/providers.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/controllers/drift_chat_controller.dart'; - -part 'providers.g.dart'; - -@Riverpod() -DriftChatController chatController(Ref ref, String chatId) { - final db = ref.watch(chatDatabaseProvider); - final controller = DriftChatController(db.messagesDao, chatId: chatId); - - ref.onDispose(() { - controller.dispose(); - }); - - return controller; -} - -@Riverpod() -Stream chatMetadata(Ref ref, String chatId) { - final db = ref.watch(chatDatabaseProvider); - return db.metadataDao.getChatMetadata(chatId).watchSingleOrNull(); -} diff --git a/app/lib/features/chat/features/chat_store/domain/providers.g.dart b/app/lib/features/chat/features/chat_store/domain/providers.g.dart deleted file mode 100644 index 0cf7db38..00000000 --- a/app/lib/features/chat/features/chat_store/domain/providers.g.dart +++ /dev/null @@ -1,273 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'providers.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$chatControllerHash() => r'493264a4413ffc2d767d73a90634e1a1744cfcfd'; - -/// Copied from Dart SDK -class _SystemHash { - _SystemHash._(); - - static int combine(int hash, int value) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + value); - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10)); - return hash ^ (hash >> 6); - } - - static int finish(int hash) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3)); - // ignore: parameter_assignments - hash = hash ^ (hash >> 11); - return 0x1fffffff & (hash + ((0x00003fff & hash) << 15)); - } -} - -/// See also [chatController]. -@ProviderFor(chatController) -const chatControllerProvider = ChatControllerFamily(); - -/// See also [chatController]. -class ChatControllerFamily extends Family { - /// See also [chatController]. - const ChatControllerFamily(); - - /// See also [chatController]. - ChatControllerProvider call(String chatId) { - return ChatControllerProvider(chatId); - } - - @override - ChatControllerProvider getProviderOverride( - covariant ChatControllerProvider provider, - ) { - return call(provider.chatId); - } - - static const Iterable? _dependencies = null; - - @override - Iterable? get dependencies => _dependencies; - - static const Iterable? _allTransitiveDependencies = null; - - @override - Iterable? get allTransitiveDependencies => - _allTransitiveDependencies; - - @override - String? get name => r'chatControllerProvider'; -} - -/// See also [chatController]. -class ChatControllerProvider extends AutoDisposeProvider { - /// See also [chatController]. - ChatControllerProvider(String chatId) - : this._internal( - (ref) => chatController(ref as ChatControllerRef, chatId), - from: chatControllerProvider, - name: r'chatControllerProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$chatControllerHash, - dependencies: ChatControllerFamily._dependencies, - allTransitiveDependencies: - ChatControllerFamily._allTransitiveDependencies, - chatId: chatId, - ); - - ChatControllerProvider._internal( - super._createNotifier, { - required super.name, - required super.dependencies, - required super.allTransitiveDependencies, - required super.debugGetCreateSourceHash, - required super.from, - required this.chatId, - }) : super.internal(); - - final String chatId; - - @override - Override overrideWith( - DriftChatController Function(ChatControllerRef provider) create, - ) { - return ProviderOverride( - origin: this, - override: ChatControllerProvider._internal( - (ref) => create(ref as ChatControllerRef), - from: from, - name: null, - dependencies: null, - allTransitiveDependencies: null, - debugGetCreateSourceHash: null, - chatId: chatId, - ), - ); - } - - @override - AutoDisposeProviderElement createElement() { - return _ChatControllerProviderElement(this); - } - - @override - bool operator ==(Object other) { - return other is ChatControllerProvider && other.chatId == chatId; - } - - @override - int get hashCode { - var hash = _SystemHash.combine(0, runtimeType.hashCode); - hash = _SystemHash.combine(hash, chatId.hashCode); - - return _SystemHash.finish(hash); - } -} - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -mixin ChatControllerRef on AutoDisposeProviderRef { - /// The parameter `chatId` of this provider. - String get chatId; -} - -class _ChatControllerProviderElement - extends AutoDisposeProviderElement - with ChatControllerRef { - _ChatControllerProviderElement(super.provider); - - @override - String get chatId => (origin as ChatControllerProvider).chatId; -} - -String _$chatMetadataHash() => r'e50f492ae69168c9f9c4df89e29424b72b681e27'; - -/// See also [chatMetadata]. -@ProviderFor(chatMetadata) -const chatMetadataProvider = ChatMetadataFamily(); - -/// See also [chatMetadata]. -class ChatMetadataFamily extends Family> { - /// See also [chatMetadata]. - const ChatMetadataFamily(); - - /// See also [chatMetadata]. - ChatMetadataProvider call(String chatId) { - return ChatMetadataProvider(chatId); - } - - @override - ChatMetadataProvider getProviderOverride( - covariant ChatMetadataProvider provider, - ) { - return call(provider.chatId); - } - - static const Iterable? _dependencies = null; - - @override - Iterable? get dependencies => _dependencies; - - static const Iterable? _allTransitiveDependencies = null; - - @override - Iterable? get allTransitiveDependencies => - _allTransitiveDependencies; - - @override - String? get name => r'chatMetadataProvider'; -} - -/// See also [chatMetadata]. -class ChatMetadataProvider extends AutoDisposeStreamProvider { - /// See also [chatMetadata]. - ChatMetadataProvider(String chatId) - : this._internal( - (ref) => chatMetadata(ref as ChatMetadataRef, chatId), - from: chatMetadataProvider, - name: r'chatMetadataProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$chatMetadataHash, - dependencies: ChatMetadataFamily._dependencies, - allTransitiveDependencies: - ChatMetadataFamily._allTransitiveDependencies, - chatId: chatId, - ); - - ChatMetadataProvider._internal( - super._createNotifier, { - required super.name, - required super.dependencies, - required super.allTransitiveDependencies, - required super.debugGetCreateSourceHash, - required super.from, - required this.chatId, - }) : super.internal(); - - final String chatId; - - @override - Override overrideWith( - Stream Function(ChatMetadataRef provider) create, - ) { - return ProviderOverride( - origin: this, - override: ChatMetadataProvider._internal( - (ref) => create(ref as ChatMetadataRef), - from: from, - name: null, - dependencies: null, - allTransitiveDependencies: null, - debugGetCreateSourceHash: null, - chatId: chatId, - ), - ); - } - - @override - AutoDisposeStreamProviderElement createElement() { - return _ChatMetadataProviderElement(this); - } - - @override - bool operator ==(Object other) { - return other is ChatMetadataProvider && other.chatId == chatId; - } - - @override - int get hashCode { - var hash = _SystemHash.combine(0, runtimeType.hashCode); - hash = _SystemHash.combine(hash, chatId.hashCode); - - return _SystemHash.finish(hash); - } -} - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -mixin ChatMetadataRef on AutoDisposeStreamProviderRef { - /// The parameter `chatId` of this provider. - String get chatId; -} - -class _ChatMetadataProviderElement - extends AutoDisposeStreamProviderElement - with ChatMetadataRef { - _ChatMetadataProviderElement(super.provider); - - @override - String get chatId => (origin as ChatMetadataProvider).chatId; -} - -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/chat/features/chat_store/domain/repositories/chat_message.dart b/app/lib/features/chat/features/chat_store/domain/repositories/chat_message.dart deleted file mode 100644 index 2e46b998..00000000 --- a/app/lib/features/chat/features/chat_store/domain/repositories/chat_message.dart +++ /dev/null @@ -1,86 +0,0 @@ -import 'package:exceptions/exceptions.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:langchain/langchain.dart'; -import 'package:nullability/nullability.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:weblibre/core/uuid.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/message_types.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/controllers/drift_chat_controller.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/providers.dart'; - -part 'chat_message.g.dart'; - -@Riverpod() -class ChatMessageRepository extends _$ChatMessageRepository { - late DriftChatController _controller; - - final _isTyping = {}; - - bool isTyping(MessageAuthor author) => _isTyping.contains(author); - - Future setTyping({ - required MessageAuthor author, - required bool typing, - }) async { - if (isTyping(author) == typing) { - return; - } - - if (typing) { - await _controller.insert( - CustomMessage( - id: uuid.v4(), - authorId: author.user.id, - createdAt: DateTime.now().toUtc(), - metadata: {'type': 'typing'}, - ), - ); - - _isTyping.add(author); - } else { - final typingMessages = _controller.messages.where( - (message) => - message.authorId == author.user.id && - message.metadata?['type'] == 'typing', - ); - - await Future.wait( - typingMessages.map((message) => _controller.remove(message)).toList(), - ); - - _isTyping.remove(author); - } - } - - Future> insertTextMessage({ - required MessageAuthor author, - required String content, - List? toolCalls, - bool hideFromModelChatHistory = false, - }) { - final message = TextMessage( - id: uuid.v4(), - authorId: author.user.id, - createdAt: DateTime.now().toUtc(), - text: content, - isOnlyEmoji: isOnlyEmoji(content), - metadata: - (hideFromModelChatHistory || (toolCalls.isNotEmpty)) - ? { - 'toolCalls': toolCalls?.map((tool) => tool.toMap()).toList(), - if (hideFromModelChatHistory) 'hideFromModelChatHistory': true, - } - : null, - ); - - return Result.fromAsync(() async { - await _controller.insert(message); - return message; - }); - } - - @override - void build(String chatId) { - _controller = ref.watch(chatControllerProvider(chatId)); - } -} diff --git a/app/lib/features/chat/features/chat_store/domain/repositories/chat_message.g.dart b/app/lib/features/chat/features/chat_store/domain/repositories/chat_message.g.dart deleted file mode 100644 index af35ad60..00000000 --- a/app/lib/features/chat/features/chat_store/domain/repositories/chat_message.g.dart +++ /dev/null @@ -1,165 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'chat_message.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$chatMessageRepositoryHash() => - r'34055588fb83fd3d4a91219e96a0c5e961af5212'; - -/// Copied from Dart SDK -class _SystemHash { - _SystemHash._(); - - static int combine(int hash, int value) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + value); - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10)); - return hash ^ (hash >> 6); - } - - static int finish(int hash) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3)); - // ignore: parameter_assignments - hash = hash ^ (hash >> 11); - return 0x1fffffff & (hash + ((0x00003fff & hash) << 15)); - } -} - -abstract class _$ChatMessageRepository - extends BuildlessAutoDisposeNotifier { - late final String chatId; - - void build(String chatId); -} - -/// See also [ChatMessageRepository]. -@ProviderFor(ChatMessageRepository) -const chatMessageRepositoryProvider = ChatMessageRepositoryFamily(); - -/// See also [ChatMessageRepository]. -class ChatMessageRepositoryFamily extends Family { - /// See also [ChatMessageRepository]. - const ChatMessageRepositoryFamily(); - - /// See also [ChatMessageRepository]. - ChatMessageRepositoryProvider call(String chatId) { - return ChatMessageRepositoryProvider(chatId); - } - - @override - ChatMessageRepositoryProvider getProviderOverride( - covariant ChatMessageRepositoryProvider provider, - ) { - return call(provider.chatId); - } - - static const Iterable? _dependencies = null; - - @override - Iterable? get dependencies => _dependencies; - - static const Iterable? _allTransitiveDependencies = null; - - @override - Iterable? get allTransitiveDependencies => - _allTransitiveDependencies; - - @override - String? get name => r'chatMessageRepositoryProvider'; -} - -/// See also [ChatMessageRepository]. -class ChatMessageRepositoryProvider - extends AutoDisposeNotifierProviderImpl { - /// See also [ChatMessageRepository]. - ChatMessageRepositoryProvider(String chatId) - : this._internal( - () => ChatMessageRepository()..chatId = chatId, - from: chatMessageRepositoryProvider, - name: r'chatMessageRepositoryProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$chatMessageRepositoryHash, - dependencies: ChatMessageRepositoryFamily._dependencies, - allTransitiveDependencies: - ChatMessageRepositoryFamily._allTransitiveDependencies, - chatId: chatId, - ); - - ChatMessageRepositoryProvider._internal( - super._createNotifier, { - required super.name, - required super.dependencies, - required super.allTransitiveDependencies, - required super.debugGetCreateSourceHash, - required super.from, - required this.chatId, - }) : super.internal(); - - final String chatId; - - @override - void runNotifierBuild(covariant ChatMessageRepository notifier) { - return notifier.build(chatId); - } - - @override - Override overrideWith(ChatMessageRepository Function() create) { - return ProviderOverride( - origin: this, - override: ChatMessageRepositoryProvider._internal( - () => create()..chatId = chatId, - from: from, - name: null, - dependencies: null, - allTransitiveDependencies: null, - debugGetCreateSourceHash: null, - chatId: chatId, - ), - ); - } - - @override - AutoDisposeNotifierProviderElement - createElement() { - return _ChatMessageRepositoryProviderElement(this); - } - - @override - bool operator ==(Object other) { - return other is ChatMessageRepositoryProvider && other.chatId == chatId; - } - - @override - int get hashCode { - var hash = _SystemHash.combine(0, runtimeType.hashCode); - hash = _SystemHash.combine(hash, chatId.hashCode); - - return _SystemHash.finish(hash); - } -} - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -mixin ChatMessageRepositoryRef on AutoDisposeNotifierProviderRef { - /// The parameter `chatId` of this provider. - String get chatId; -} - -class _ChatMessageRepositoryProviderElement - extends AutoDisposeNotifierProviderElement - with ChatMessageRepositoryRef { - _ChatMessageRepositoryProviderElement(super.provider); - - @override - String get chatId => (origin as ChatMessageRepositoryProvider).chatId; -} - -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.dart b/app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.dart deleted file mode 100644 index 38cf8a38..00000000 --- a/app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.dart +++ /dev/null @@ -1,21 +0,0 @@ -import 'package:exceptions/exceptions.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/providers.dart'; - -part 'chat_metadata.g.dart'; - -@Riverpod() -class ChatMetadataRepository extends _$ChatMetadataRepository { - Future> updateMetadata(ChatMetadata metadata) { - return Result.fromAsync(() { - return ref - .read(chatDatabaseProvider) - .metadataDao - .updateChatMetadata(chatId, metadata); - }); - } - - @override - void build(String chatId) {} -} diff --git a/app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.g.dart b/app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.g.dart deleted file mode 100644 index 2df00afe..00000000 --- a/app/lib/features/chat/features/chat_store/domain/repositories/chat_metadata.g.dart +++ /dev/null @@ -1,165 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'chat_metadata.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$chatMetadataRepositoryHash() => - r'bd6b28b9d034ee82f95e974e4bf8f553546e835a'; - -/// Copied from Dart SDK -class _SystemHash { - _SystemHash._(); - - static int combine(int hash, int value) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + value); - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10)); - return hash ^ (hash >> 6); - } - - static int finish(int hash) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3)); - // ignore: parameter_assignments - hash = hash ^ (hash >> 11); - return 0x1fffffff & (hash + ((0x00003fff & hash) << 15)); - } -} - -abstract class _$ChatMetadataRepository - extends BuildlessAutoDisposeNotifier { - late final String chatId; - - void build(String chatId); -} - -/// See also [ChatMetadataRepository]. -@ProviderFor(ChatMetadataRepository) -const chatMetadataRepositoryProvider = ChatMetadataRepositoryFamily(); - -/// See also [ChatMetadataRepository]. -class ChatMetadataRepositoryFamily extends Family { - /// See also [ChatMetadataRepository]. - const ChatMetadataRepositoryFamily(); - - /// See also [ChatMetadataRepository]. - ChatMetadataRepositoryProvider call(String chatId) { - return ChatMetadataRepositoryProvider(chatId); - } - - @override - ChatMetadataRepositoryProvider getProviderOverride( - covariant ChatMetadataRepositoryProvider provider, - ) { - return call(provider.chatId); - } - - static const Iterable? _dependencies = null; - - @override - Iterable? get dependencies => _dependencies; - - static const Iterable? _allTransitiveDependencies = null; - - @override - Iterable? get allTransitiveDependencies => - _allTransitiveDependencies; - - @override - String? get name => r'chatMetadataRepositoryProvider'; -} - -/// See also [ChatMetadataRepository]. -class ChatMetadataRepositoryProvider - extends AutoDisposeNotifierProviderImpl { - /// See also [ChatMetadataRepository]. - ChatMetadataRepositoryProvider(String chatId) - : this._internal( - () => ChatMetadataRepository()..chatId = chatId, - from: chatMetadataRepositoryProvider, - name: r'chatMetadataRepositoryProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$chatMetadataRepositoryHash, - dependencies: ChatMetadataRepositoryFamily._dependencies, - allTransitiveDependencies: - ChatMetadataRepositoryFamily._allTransitiveDependencies, - chatId: chatId, - ); - - ChatMetadataRepositoryProvider._internal( - super._createNotifier, { - required super.name, - required super.dependencies, - required super.allTransitiveDependencies, - required super.debugGetCreateSourceHash, - required super.from, - required this.chatId, - }) : super.internal(); - - final String chatId; - - @override - void runNotifierBuild(covariant ChatMetadataRepository notifier) { - return notifier.build(chatId); - } - - @override - Override overrideWith(ChatMetadataRepository Function() create) { - return ProviderOverride( - origin: this, - override: ChatMetadataRepositoryProvider._internal( - () => create()..chatId = chatId, - from: from, - name: null, - dependencies: null, - allTransitiveDependencies: null, - debugGetCreateSourceHash: null, - chatId: chatId, - ), - ); - } - - @override - AutoDisposeNotifierProviderElement - createElement() { - return _ChatMetadataRepositoryProviderElement(this); - } - - @override - bool operator ==(Object other) { - return other is ChatMetadataRepositoryProvider && other.chatId == chatId; - } - - @override - int get hashCode { - var hash = _SystemHash.combine(0, runtimeType.hashCode); - hash = _SystemHash.combine(hash, chatId.hashCode); - - return _SystemHash.finish(hash); - } -} - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -mixin ChatMetadataRepositoryRef on AutoDisposeNotifierProviderRef { - /// The parameter `chatId` of this provider. - String get chatId; -} - -class _ChatMetadataRepositoryProviderElement - extends AutoDisposeNotifierProviderElement - with ChatMetadataRepositoryRef { - _ChatMetadataRepositoryProviderElement(super.provider); - - @override - String get chatId => (origin as ChatMetadataRepositoryProvider).chatId; -} - -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/geckoview/features/browser/presentation/dialogs/web_page_dialog.dart b/app/lib/features/geckoview/features/browser/presentation/dialogs/web_page_dialog.dart index a59f8c20..6733c84b 100644 --- a/app/lib/features/geckoview/features/browser/presentation/dialogs/web_page_dialog.dart +++ b/app/lib/features/geckoview/features/browser/presentation/dialogs/web_page_dialog.dart @@ -10,13 +10,8 @@ import 'package:share_plus/share_plus.dart'; import 'package:weblibre/data/models/web_page_info.dart'; import 'package:weblibre/features/bangs/domain/providers/bangs.dart'; import 'package:weblibre/features/bangs/presentation/widgets/site_search.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/repositories/chat_metadata.dart'; -import 'package:weblibre/features/geckoview/domain/controllers/bottom_sheet.dart'; import 'package:weblibre/features/geckoview/domain/providers/tab_session.dart'; -import 'package:weblibre/features/geckoview/domain/providers/tab_state.dart'; import 'package:weblibre/features/geckoview/domain/repositories/tab.dart'; -import 'package:weblibre/features/geckoview/features/browser/domain/entities/sheet.dart'; import 'package:weblibre/features/geckoview/features/browser/presentation/widgets/browser_modules/address_with_suggestions_field.dart'; import 'package:weblibre/features/user/domain/providers.dart'; import 'package:weblibre/presentation/widgets/failure_widget.dart'; @@ -236,59 +231,6 @@ class WebPageDialog extends HookConsumerWidget { } }, ), - const Divider(), - ListTile( - leading: const Icon(MdiIcons.brain), - title: const Text('QA Chat'), - onTap: () async { - final selectedTabId = - ref.read(selectedTabStateProvider)?.id; - - if (selectedTabId != null) { - final updateResult = await ref - .read( - chatMetadataRepositoryProvider( - selectedTabId, - ).notifier, - ) - .updateMetadata( - ChatMetadata(mainDocumentId: selectedTabId), - ); - - updateResult.onSuccess((_) { - ref - .read( - bottomSheetControllerProvider.notifier, - ) - .show( - TabQaChatSheet(chatId: selectedTabId), - ); - }); - } - - if (context.mounted) { - context.pop(); - } - }, - ), - // ListTile( - // leading: Icon(MdiIcons.text), - // title: const Text('Summarize'), - // onTap: () async { - // final summarizerUrl = uri_builder.summarizerUri( - // document: SharedUrl(url), - // mode: SummarizerMode.keyMoments, - // ); - - // await ref - // .read(tabRepositoryProvider.notifier) - // .addTab(url: summarizerUrl); - - // if (context.mounted) { - // context.pop(); - // } - // }, - // ), ], ), ); diff --git a/app/lib/features/geckoview/features/browser/presentation/screens/browser.dart b/app/lib/features/geckoview/features/browser/presentation/screens/browser.dart index 5469e31a..83a24eb9 100644 --- a/app/lib/features/geckoview/features/browser/presentation/screens/browser.dart +++ b/app/lib/features/geckoview/features/browser/presentation/screens/browser.dart @@ -24,7 +24,6 @@ import 'package:weblibre/features/geckoview/features/browser/presentation/widget import 'package:weblibre/features/geckoview/features/contextmenu/extensions/hit_result.dart'; import 'package:weblibre/features/geckoview/features/find_in_page/presentation/widgets/find_in_page.dart'; import 'package:weblibre/features/geckoview/features/readerview/presentation/widgets/reader_appearance_button.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/chat/presentation/widgets/tab_qa_chat.dart'; import 'package:weblibre/utils/ui_helper.dart' as ui_helper; class BrowserScreen extends HookConsumerWidget { @@ -371,12 +370,6 @@ class _QaSheet extends HookConsumerWidget { ), ), ), - Expanded( - child: TabQaChat( - chatId: chatId, - scrollController: scrollController, - ), - ), ], ), ); diff --git a/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart b/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart index 71b61677..7c0274ee 100644 --- a/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart +++ b/app/lib/features/geckoview/features/browser/presentation/widgets/browser_modules/browser_view.dart @@ -19,7 +19,6 @@ import 'package:weblibre/features/geckoview/features/browser/domain/providers/li import 'package:weblibre/features/geckoview/features/browser/domain/services/browser_data.dart'; import 'package:weblibre/features/geckoview/features/browser/domain/services/engine_settings_replication.dart'; import 'package:weblibre/features/geckoview/features/browser/domain/services/proxy_settings_replication.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart'; import 'package:weblibre/features/user/domain/repositories/cache.dart'; import 'package:weblibre/features/user/domain/repositories/general_settings.dart'; import 'package:weblibre/features/user/domain/services/local_authentication.dart'; @@ -154,7 +153,6 @@ class _BrowserViewState extends ConsumerState //Initialize and register dependencies ref.listenManual(tabRepositoryProvider, (previous, next) {}); - ref.listenManual(documentRepositoryProvider, (previous, next) {}); ref.listenManual(selectionActionServiceProvider, (previous, next) {}); diff --git a/app/lib/features/geckoview/features/tabs/data/database/database.dart b/app/lib/features/geckoview/features/tabs/data/database/database.dart index 8bb65963..5d261e67 100644 --- a/app/lib/features/geckoview/features/tabs/data/database/database.dart +++ b/app/lib/features/geckoview/features/tabs/data/database/database.dart @@ -7,20 +7,12 @@ import 'package:weblibre/features/geckoview/features/tabs/data/database/daos/con import 'package:weblibre/features/geckoview/features/tabs/data/database/daos/tab.dart'; import 'package:weblibre/features/geckoview/features/tabs/data/models/container_data.dart'; import 'package:weblibre/features/geckoview/features/tabs/data/models/tab_query_result.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/migrator.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart'; import 'package:weblibre/features/search/domain/fts_tokenizer.dart'; part 'database.g.dart'; -@DriftDatabase( - include: {'database.drift'}, - daos: [ContainerDao, TabDao, VectorDao], -) +@DriftDatabase(include: {'database.drift'}, daos: [ContainerDao, TabDao]) class TabDatabase extends _$TabDatabase with TrigramQueryBuilderMixin { - final int embeddingDimensions; - @override final int schemaVersion = 2; @@ -31,23 +23,11 @@ class TabDatabase extends _$TabDatabase with TrigramQueryBuilderMixin { @override MigrationStrategy get migration => MigrationStrategy( - onCreate: (m) async { - final migrator = VectorDatabaseMigrator(dimensions: embeddingDimensions); - - await m.database.customStatement(migrator.vectorTableDefinition); - - //instead of m.createAll(); we igoner vec0 table - for (final entity in allSchemaEntities.where( - (entity) => entity is! DocumentVec, - )) { - await m.create(entity); - } - }, beforeOpen: (details) async { await customStatement('PRAGMA foreign_keys = ON;'); await optimizeFtsIndex(); }, ); - TabDatabase(super.e, {required this.embeddingDimensions}); + TabDatabase(super.e); } diff --git a/app/lib/features/geckoview/features/tabs/data/database/database.drift b/app/lib/features/geckoview/features/tabs/data/database/database.drift index a8e53284..04e53900 100644 --- a/app/lib/features/geckoview/features/tabs/data/database/database.drift +++ b/app/lib/features/geckoview/features/tabs/data/database/database.drift @@ -25,14 +25,6 @@ CREATE TABLE tab ( timestamp DATETIME NOT NULL ); -import '../../features/vector_store/data/database/vector_store.drift'; - --- automativcally remove documents and embeddings on delete --- isnert and updates are managed code side -CREATE TRIGGER tab_document_delete AFTER DELETE ON tab BEGIN - DELETE FROM document WHERE main_document_id = old.id; -END; - CREATE VIRTUAL TABLE tab_fts USING fts5( title, diff --git a/app/lib/features/geckoview/features/tabs/data/database/database.g.dart b/app/lib/features/geckoview/features/tabs/data/database/database.g.dart index 4efcaf48..25106ec7 100644 --- a/app/lib/features/geckoview/features/tabs/data/database/database.g.dart +++ b/app/lib/features/geckoview/features/tabs/data/database/database.g.dart @@ -781,354 +781,6 @@ class TabCompanion extends UpdateCompanion { } } -class Document extends Table with TableInfo { - @override - final GeneratedDatabase attachedDatabase; - final String? _alias; - Document(this.attachedDatabase, [this._alias]); - late final GeneratedColumn documentId = GeneratedColumn( - 'document_id', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL PRIMARY KEY', - ); - late final GeneratedColumn mainDocumentId = GeneratedColumn( - 'main_document_id', - aliasedName, - true, - type: DriftSqlType.string, - requiredDuringInsert: false, - $customConstraints: '', - ); - late final GeneratedColumn contextId = GeneratedColumn( - 'context_id', - aliasedName, - true, - type: DriftSqlType.string, - requiredDuringInsert: false, - $customConstraints: '', - ); - late final GeneratedColumn content = GeneratedColumn( - 'content', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - late final GeneratedColumn contentHash = GeneratedColumn( - 'content_hash', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - late final GeneratedColumn metadata = GeneratedColumn( - 'metadata', - aliasedName, - true, - type: DriftSqlType.string, - requiredDuringInsert: false, - $customConstraints: '', - ); - @override - List get $columns => [ - documentId, - mainDocumentId, - contextId, - content, - contentHash, - metadata, - ]; - @override - String get aliasedName => _alias ?? actualTableName; - @override - String get actualTableName => $name; - static const String $name = 'document'; - @override - Set get $primaryKey => {documentId}; - @override - DocumentData map(Map data, {String? tablePrefix}) { - final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; - return DocumentData( - documentId: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}document_id'], - )!, - mainDocumentId: attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}main_document_id'], - ), - contextId: attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}context_id'], - ), - content: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}content'], - )!, - contentHash: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}content_hash'], - )!, - metadata: attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}metadata'], - ), - ); - } - - @override - Document createAlias(String alias) { - return Document(attachedDatabase, alias); - } - - @override - bool get dontWriteConstraints => true; -} - -class DocumentData extends DataClass implements Insertable { - final String documentId; - final String? mainDocumentId; - final String? contextId; - final String content; - final String contentHash; - final String? metadata; - const DocumentData({ - required this.documentId, - this.mainDocumentId, - this.contextId, - required this.content, - required this.contentHash, - this.metadata, - }); - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - map['document_id'] = Variable(documentId); - if (!nullToAbsent || mainDocumentId != null) { - map['main_document_id'] = Variable(mainDocumentId); - } - if (!nullToAbsent || contextId != null) { - map['context_id'] = Variable(contextId); - } - map['content'] = Variable(content); - map['content_hash'] = Variable(contentHash); - if (!nullToAbsent || metadata != null) { - map['metadata'] = Variable(metadata); - } - return map; - } - - factory DocumentData.fromJson( - Map json, { - ValueSerializer? serializer, - }) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return DocumentData( - documentId: serializer.fromJson(json['document_id']), - mainDocumentId: serializer.fromJson(json['main_document_id']), - contextId: serializer.fromJson(json['context_id']), - content: serializer.fromJson(json['content']), - contentHash: serializer.fromJson(json['content_hash']), - metadata: serializer.fromJson(json['metadata']), - ); - } - @override - Map toJson({ValueSerializer? serializer}) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return { - 'document_id': serializer.toJson(documentId), - 'main_document_id': serializer.toJson(mainDocumentId), - 'context_id': serializer.toJson(contextId), - 'content': serializer.toJson(content), - 'content_hash': serializer.toJson(contentHash), - 'metadata': serializer.toJson(metadata), - }; - } - - DocumentData copyWith({ - String? documentId, - Value mainDocumentId = const Value.absent(), - Value contextId = const Value.absent(), - String? content, - String? contentHash, - Value metadata = const Value.absent(), - }) => DocumentData( - documentId: documentId ?? this.documentId, - mainDocumentId: - mainDocumentId.present ? mainDocumentId.value : this.mainDocumentId, - contextId: contextId.present ? contextId.value : this.contextId, - content: content ?? this.content, - contentHash: contentHash ?? this.contentHash, - metadata: metadata.present ? metadata.value : this.metadata, - ); - DocumentData copyWithCompanion(DocumentCompanion data) { - return DocumentData( - documentId: - data.documentId.present ? data.documentId.value : this.documentId, - mainDocumentId: - data.mainDocumentId.present - ? data.mainDocumentId.value - : this.mainDocumentId, - contextId: data.contextId.present ? data.contextId.value : this.contextId, - content: data.content.present ? data.content.value : this.content, - contentHash: - data.contentHash.present ? data.contentHash.value : this.contentHash, - metadata: data.metadata.present ? data.metadata.value : this.metadata, - ); - } - - @override - String toString() { - return (StringBuffer('DocumentData(') - ..write('documentId: $documentId, ') - ..write('mainDocumentId: $mainDocumentId, ') - ..write('contextId: $contextId, ') - ..write('content: $content, ') - ..write('contentHash: $contentHash, ') - ..write('metadata: $metadata') - ..write(')')) - .toString(); - } - - @override - int get hashCode => Object.hash( - documentId, - mainDocumentId, - contextId, - content, - contentHash, - metadata, - ); - @override - bool operator ==(Object other) => - identical(this, other) || - (other is DocumentData && - other.documentId == this.documentId && - other.mainDocumentId == this.mainDocumentId && - other.contextId == this.contextId && - other.content == this.content && - other.contentHash == this.contentHash && - other.metadata == this.metadata); -} - -class DocumentCompanion extends UpdateCompanion { - final Value documentId; - final Value mainDocumentId; - final Value contextId; - final Value content; - final Value contentHash; - final Value metadata; - final Value rowid; - const DocumentCompanion({ - this.documentId = const Value.absent(), - this.mainDocumentId = const Value.absent(), - this.contextId = const Value.absent(), - this.content = const Value.absent(), - this.contentHash = const Value.absent(), - this.metadata = const Value.absent(), - this.rowid = const Value.absent(), - }); - DocumentCompanion.insert({ - required String documentId, - this.mainDocumentId = const Value.absent(), - this.contextId = const Value.absent(), - required String content, - required String contentHash, - this.metadata = const Value.absent(), - this.rowid = const Value.absent(), - }) : documentId = Value(documentId), - content = Value(content), - contentHash = Value(contentHash); - static Insertable custom({ - Expression? documentId, - Expression? mainDocumentId, - Expression? contextId, - Expression? content, - Expression? contentHash, - Expression? metadata, - Expression? rowid, - }) { - return RawValuesInsertable({ - if (documentId != null) 'document_id': documentId, - if (mainDocumentId != null) 'main_document_id': mainDocumentId, - if (contextId != null) 'context_id': contextId, - if (content != null) 'content': content, - if (contentHash != null) 'content_hash': contentHash, - if (metadata != null) 'metadata': metadata, - if (rowid != null) 'rowid': rowid, - }); - } - - DocumentCompanion copyWith({ - Value? documentId, - Value? mainDocumentId, - Value? contextId, - Value? content, - Value? contentHash, - Value? metadata, - Value? rowid, - }) { - return DocumentCompanion( - documentId: documentId ?? this.documentId, - mainDocumentId: mainDocumentId ?? this.mainDocumentId, - contextId: contextId ?? this.contextId, - content: content ?? this.content, - contentHash: contentHash ?? this.contentHash, - metadata: metadata ?? this.metadata, - rowid: rowid ?? this.rowid, - ); - } - - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - if (documentId.present) { - map['document_id'] = Variable(documentId.value); - } - if (mainDocumentId.present) { - map['main_document_id'] = Variable(mainDocumentId.value); - } - if (contextId.present) { - map['context_id'] = Variable(contextId.value); - } - if (content.present) { - map['content'] = Variable(content.value); - } - if (contentHash.present) { - map['content_hash'] = Variable(contentHash.value); - } - if (metadata.present) { - map['metadata'] = Variable(metadata.value); - } - if (rowid.present) { - map['rowid'] = Variable(rowid.value); - } - return map; - } - - @override - String toString() { - return (StringBuffer('DocumentCompanion(') - ..write('documentId: $documentId, ') - ..write('mainDocumentId: $mainDocumentId, ') - ..write('contextId: $contextId, ') - ..write('content: $content, ') - ..write('contentHash: $contentHash, ') - ..write('metadata: $metadata, ') - ..write('rowid: $rowid') - ..write(')')) - .toString(); - } -} - class TabFts extends Table with TableInfo, VirtualTableInfo { @override @@ -1412,402 +1064,11 @@ class TabFtsCompanion extends UpdateCompanion { } } -class DocumentVec extends Table with TableInfo { - @override - final GeneratedDatabase attachedDatabase; - final String? _alias; - DocumentVec(this.attachedDatabase, [this._alias]); - late final GeneratedColumn id = GeneratedColumn( - 'id', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL PRIMARY KEY', - ); - late final GeneratedColumn mainDocumentId = GeneratedColumn( - 'main_document_id', - aliasedName, - true, - type: DriftSqlType.string, - requiredDuringInsert: false, - $customConstraints: '', - ); - late final GeneratedColumn contextId = GeneratedColumn( - 'context_id', - aliasedName, - true, - type: DriftSqlType.string, - requiredDuringInsert: false, - $customConstraints: '', - ); - late final GeneratedColumn embedding = GeneratedColumn( - 'embedding', - aliasedName, - false, - type: DriftSqlType.blob, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - late final GeneratedColumn contentHash = GeneratedColumn( - 'content_hash', - aliasedName, - false, - type: DriftSqlType.string, - requiredDuringInsert: true, - $customConstraints: 'NOT NULL', - ); - late final GeneratedColumn distance = GeneratedColumn( - 'distance', - aliasedName, - true, - type: DriftSqlType.double, - requiredDuringInsert: false, - $customConstraints: '', - ); - late final GeneratedColumn k = GeneratedColumn( - 'k', - aliasedName, - true, - type: DriftSqlType.int, - requiredDuringInsert: false, - $customConstraints: '', - ); - @override - List get $columns => [ - id, - mainDocumentId, - contextId, - embedding, - contentHash, - distance, - k, - ]; - @override - String get aliasedName => _alias ?? actualTableName; - @override - String get actualTableName => $name; - static const String $name = 'document_vec'; - @override - Set get $primaryKey => {id}; - @override - DocumentVecData map(Map data, {String? tablePrefix}) { - final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; - return DocumentVecData( - id: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}id'], - )!, - mainDocumentId: attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}main_document_id'], - ), - contextId: attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}context_id'], - ), - embedding: - attachedDatabase.typeMapping.read( - DriftSqlType.blob, - data['${effectivePrefix}embedding'], - )!, - contentHash: - attachedDatabase.typeMapping.read( - DriftSqlType.string, - data['${effectivePrefix}content_hash'], - )!, - distance: attachedDatabase.typeMapping.read( - DriftSqlType.double, - data['${effectivePrefix}distance'], - ), - k: attachedDatabase.typeMapping.read( - DriftSqlType.int, - data['${effectivePrefix}k'], - ), - ); - } - - @override - DocumentVec createAlias(String alias) { - return DocumentVec(attachedDatabase, alias); - } - - @override - bool get dontWriteConstraints => true; -} - -class DocumentVecData extends DataClass implements Insertable { - /// dummy definition of virtual table - /// keep in sync with defined table definition - final String id; - final String? mainDocumentId; - final String? contextId; - final Uint8List embedding; - final String contentHash; - final double? distance; - final int? k; - const DocumentVecData({ - required this.id, - this.mainDocumentId, - this.contextId, - required this.embedding, - required this.contentHash, - this.distance, - this.k, - }); - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - map['id'] = Variable(id); - if (!nullToAbsent || mainDocumentId != null) { - map['main_document_id'] = Variable(mainDocumentId); - } - if (!nullToAbsent || contextId != null) { - map['context_id'] = Variable(contextId); - } - map['embedding'] = Variable(embedding); - map['content_hash'] = Variable(contentHash); - if (!nullToAbsent || distance != null) { - map['distance'] = Variable(distance); - } - if (!nullToAbsent || k != null) { - map['k'] = Variable(k); - } - return map; - } - - factory DocumentVecData.fromJson( - Map json, { - ValueSerializer? serializer, - }) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return DocumentVecData( - id: serializer.fromJson(json['id']), - mainDocumentId: serializer.fromJson(json['main_document_id']), - contextId: serializer.fromJson(json['context_id']), - embedding: serializer.fromJson(json['embedding']), - contentHash: serializer.fromJson(json['content_hash']), - distance: serializer.fromJson(json['distance']), - k: serializer.fromJson(json['k']), - ); - } - @override - Map toJson({ValueSerializer? serializer}) { - serializer ??= driftRuntimeOptions.defaultSerializer; - return { - 'id': serializer.toJson(id), - 'main_document_id': serializer.toJson(mainDocumentId), - 'context_id': serializer.toJson(contextId), - 'embedding': serializer.toJson(embedding), - 'content_hash': serializer.toJson(contentHash), - 'distance': serializer.toJson(distance), - 'k': serializer.toJson(k), - }; - } - - DocumentVecData copyWith({ - String? id, - Value mainDocumentId = const Value.absent(), - Value contextId = const Value.absent(), - Uint8List? embedding, - String? contentHash, - Value distance = const Value.absent(), - Value k = const Value.absent(), - }) => DocumentVecData( - id: id ?? this.id, - mainDocumentId: - mainDocumentId.present ? mainDocumentId.value : this.mainDocumentId, - contextId: contextId.present ? contextId.value : this.contextId, - embedding: embedding ?? this.embedding, - contentHash: contentHash ?? this.contentHash, - distance: distance.present ? distance.value : this.distance, - k: k.present ? k.value : this.k, - ); - DocumentVecData copyWithCompanion(DocumentVecCompanion data) { - return DocumentVecData( - id: data.id.present ? data.id.value : this.id, - mainDocumentId: - data.mainDocumentId.present - ? data.mainDocumentId.value - : this.mainDocumentId, - contextId: data.contextId.present ? data.contextId.value : this.contextId, - embedding: data.embedding.present ? data.embedding.value : this.embedding, - contentHash: - data.contentHash.present ? data.contentHash.value : this.contentHash, - distance: data.distance.present ? data.distance.value : this.distance, - k: data.k.present ? data.k.value : this.k, - ); - } - - @override - String toString() { - return (StringBuffer('DocumentVecData(') - ..write('id: $id, ') - ..write('mainDocumentId: $mainDocumentId, ') - ..write('contextId: $contextId, ') - ..write('embedding: $embedding, ') - ..write('contentHash: $contentHash, ') - ..write('distance: $distance, ') - ..write('k: $k') - ..write(')')) - .toString(); - } - - @override - int get hashCode => Object.hash( - id, - mainDocumentId, - contextId, - $driftBlobEquality.hash(embedding), - contentHash, - distance, - k, - ); - @override - bool operator ==(Object other) => - identical(this, other) || - (other is DocumentVecData && - other.id == this.id && - other.mainDocumentId == this.mainDocumentId && - other.contextId == this.contextId && - $driftBlobEquality.equals(other.embedding, this.embedding) && - other.contentHash == this.contentHash && - other.distance == this.distance && - other.k == this.k); -} - -class DocumentVecCompanion extends UpdateCompanion { - final Value id; - final Value mainDocumentId; - final Value contextId; - final Value embedding; - final Value contentHash; - final Value distance; - final Value k; - final Value rowid; - const DocumentVecCompanion({ - this.id = const Value.absent(), - this.mainDocumentId = const Value.absent(), - this.contextId = const Value.absent(), - this.embedding = const Value.absent(), - this.contentHash = const Value.absent(), - this.distance = const Value.absent(), - this.k = const Value.absent(), - this.rowid = const Value.absent(), - }); - DocumentVecCompanion.insert({ - required String id, - this.mainDocumentId = const Value.absent(), - this.contextId = const Value.absent(), - required Uint8List embedding, - required String contentHash, - this.distance = const Value.absent(), - this.k = const Value.absent(), - this.rowid = const Value.absent(), - }) : id = Value(id), - embedding = Value(embedding), - contentHash = Value(contentHash); - static Insertable custom({ - Expression? id, - Expression? mainDocumentId, - Expression? contextId, - Expression? embedding, - Expression? contentHash, - Expression? distance, - Expression? k, - Expression? rowid, - }) { - return RawValuesInsertable({ - if (id != null) 'id': id, - if (mainDocumentId != null) 'main_document_id': mainDocumentId, - if (contextId != null) 'context_id': contextId, - if (embedding != null) 'embedding': embedding, - if (contentHash != null) 'content_hash': contentHash, - if (distance != null) 'distance': distance, - if (k != null) 'k': k, - if (rowid != null) 'rowid': rowid, - }); - } - - DocumentVecCompanion copyWith({ - Value? id, - Value? mainDocumentId, - Value? contextId, - Value? embedding, - Value? contentHash, - Value? distance, - Value? k, - Value? rowid, - }) { - return DocumentVecCompanion( - id: id ?? this.id, - mainDocumentId: mainDocumentId ?? this.mainDocumentId, - contextId: contextId ?? this.contextId, - embedding: embedding ?? this.embedding, - contentHash: contentHash ?? this.contentHash, - distance: distance ?? this.distance, - k: k ?? this.k, - rowid: rowid ?? this.rowid, - ); - } - - @override - Map toColumns(bool nullToAbsent) { - final map = {}; - if (id.present) { - map['id'] = Variable(id.value); - } - if (mainDocumentId.present) { - map['main_document_id'] = Variable(mainDocumentId.value); - } - if (contextId.present) { - map['context_id'] = Variable(contextId.value); - } - if (embedding.present) { - map['embedding'] = Variable(embedding.value); - } - if (contentHash.present) { - map['content_hash'] = Variable(contentHash.value); - } - if (distance.present) { - map['distance'] = Variable(distance.value); - } - if (k.present) { - map['k'] = Variable(k.value); - } - if (rowid.present) { - map['rowid'] = Variable(rowid.value); - } - return map; - } - - @override - String toString() { - return (StringBuffer('DocumentVecCompanion(') - ..write('id: $id, ') - ..write('mainDocumentId: $mainDocumentId, ') - ..write('contextId: $contextId, ') - ..write('embedding: $embedding, ') - ..write('contentHash: $contentHash, ') - ..write('distance: $distance, ') - ..write('k: $k, ') - ..write('rowid: $rowid') - ..write(')')) - .toString(); - } -} - abstract class _$TabDatabase extends GeneratedDatabase { _$TabDatabase(QueryExecutor e) : super(e); $TabDatabaseManager get managers => $TabDatabaseManager(this); late final Container container = Container(this); late final Tab tab = Tab(this); - late final Document document = Document(this); - late final Trigger tabDocumentDelete = Trigger( - 'CREATE TRIGGER tab_document_delete AFTER DELETE ON tab BEGIN DELETE FROM document WHERE main_document_id = old.id;END', - 'tab_document_delete', - ); late final TabFts tabFts = TabFts(this); late final Trigger tabAfterInsert = Trigger( 'CREATE TRIGGER tab_after_insert AFTER INSERT ON tab BEGIN INSERT INTO tab_fts ("rowid", title, url, extracted_content_plain, full_content_plain) VALUES (new."rowid", new.title, new.url, new.extracted_content_plain, new.full_content_plain);END', @@ -1821,26 +1082,8 @@ abstract class _$TabDatabase extends GeneratedDatabase { 'CREATE TRIGGER tab_after_update AFTER UPDATE ON tab BEGIN INSERT INTO tab_fts (tab_fts, "rowid", title, url, extracted_content_plain, full_content_plain) VALUES (\'delete\', old."rowid", old.title, old.url, old.extracted_content_plain, old.full_content_plain);INSERT INTO tab_fts ("rowid", title, url, extracted_content_plain, full_content_plain) VALUES (new."rowid", new.title, new.url, new.extracted_content_plain, new.full_content_plain);END', 'tab_after_update', ); - late final Index documentMainDocumentId = Index( - 'document_main_document_id', - 'CREATE INDEX document_main_document_id ON document (main_document_id)', - ); - late final Index documentContextId = Index( - 'document_context_id', - 'CREATE INDEX document_context_id ON document (context_id)', - ); - late final DocumentVec documentVec = DocumentVec(this); - late final Trigger documentDelete = Trigger( - 'CREATE TRIGGER document_delete AFTER DELETE ON document BEGIN DELETE FROM document_vec WHERE id = old.document_id;END', - 'document_delete', - ); - late final Trigger documentUpdateDelete = Trigger( - 'CREATE TRIGGER document_update_delete AFTER UPDATE ON document BEGIN DELETE FROM document_vec WHERE id = new.document_id AND content_hash != new.content_hash;END', - 'document_update_delete', - ); late final ContainerDao containerDao = ContainerDao(this as TabDatabase); late final TabDao tabDao = TabDao(this as TabDatabase); - late final VectorDao vectorDao = VectorDao(this as TabDatabase); Future optimizeFtsIndex() { return customInsert( 'INSERT INTO tab_fts (tab_fts) VALUES (\'optimize\')', @@ -1953,47 +1196,6 @@ abstract class _$TabDatabase extends GeneratedDatabase { ); } - Selectable missingDocumentEmbeddings({ - String? mainDocumentId, - String? contextId, - }) { - return customSelect( - 'SELECT doc.* FROM document AS doc WHERE doc.main_document_id IS COALESCE(?1, doc.main_document_id) AND doc.context_id IS COALESCE(?2, doc.context_id) AND NOT EXISTS (SELECT 1 AS _c0 FROM document_vec AS vec WHERE vec.id = doc.document_id)', - variables: [ - Variable(mainDocumentId), - Variable(contextId), - ], - readsFrom: {document, documentVec}, - ).asyncMap(document.mapFromRow); - } - - Selectable queryVectors({ - required Uint8List searchVectors, - int? k, - String? mainDocumentId, - String? contextId, - }) { - return customSelect( - 'SELECT vec.id, doc.main_document_id, doc.context_id, doc.content, doc.metadata, vec.distance FROM document_vec AS vec INNER JOIN document AS doc ON doc.document_id = vec.id WHERE vec.embedding MATCH ?1 AND vec.k = ?2 AND vec.main_document_id IS COALESCE(?3, vec.main_document_id) AND vec.context_id IS COALESCE(?4, vec.context_id) ORDER BY vec.distance', - variables: [ - Variable(searchVectors), - Variable(k), - Variable(mainDocumentId), - Variable(contextId), - ], - readsFrom: {documentVec, document}, - ).map( - (QueryRow row) => VectorResult( - id: row.read('id'), - mainDocumentId: row.readNullable('main_document_id'), - contextId: row.readNullable('context_id'), - content: row.read('content'), - metadata: row.readNullable('metadata'), - distance: row.readNullable('distance'), - ), - ); - } - @override Iterable> get allTables => allSchemaEntities.whereType>(); @@ -2001,17 +1203,10 @@ abstract class _$TabDatabase extends GeneratedDatabase { List get allSchemaEntities => [ container, tab, - document, - tabDocumentDelete, tabFts, tabAfterInsert, tabAfterDelete, tabAfterUpdate, - documentMainDocumentId, - documentContextId, - documentVec, - documentDelete, - documentUpdateDelete, ]; @override StreamQueryUpdateRules get streamUpdateRules => const StreamQueryUpdateRules([ @@ -2022,13 +1217,6 @@ abstract class _$TabDatabase extends GeneratedDatabase { ), result: [TableUpdate('tab', kind: UpdateKind.delete)], ), - WritePropagation( - on: TableUpdateQuery.onTableName( - 'tab', - limitUpdateKind: UpdateKind.delete, - ), - result: [TableUpdate('document', kind: UpdateKind.delete)], - ), WritePropagation( on: TableUpdateQuery.onTableName( 'tab', @@ -2050,20 +1238,6 @@ abstract class _$TabDatabase extends GeneratedDatabase { ), result: [TableUpdate('tab_fts', kind: UpdateKind.insert)], ), - WritePropagation( - on: TableUpdateQuery.onTableName( - 'document', - limitUpdateKind: UpdateKind.delete, - ), - result: [TableUpdate('document_vec', kind: UpdateKind.delete)], - ), - WritePropagation( - on: TableUpdateQuery.onTableName( - 'document', - limitUpdateKind: UpdateKind.update, - ), - result: [TableUpdate('document_vec', kind: UpdateKind.delete)], - ), ]); } @@ -2785,229 +1959,6 @@ typedef $TabProcessedTableManager = TabData, PrefetchHooks Function({bool containerId}) >; -typedef $DocumentCreateCompanionBuilder = - DocumentCompanion Function({ - required String documentId, - Value mainDocumentId, - Value contextId, - required String content, - required String contentHash, - Value metadata, - Value rowid, - }); -typedef $DocumentUpdateCompanionBuilder = - DocumentCompanion Function({ - Value documentId, - Value mainDocumentId, - Value contextId, - Value content, - Value contentHash, - Value metadata, - Value rowid, - }); - -class $DocumentFilterComposer extends Composer<_$TabDatabase, Document> { - $DocumentFilterComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnFilters get documentId => $composableBuilder( - column: $table.documentId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get mainDocumentId => $composableBuilder( - column: $table.mainDocumentId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get contextId => $composableBuilder( - column: $table.contextId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get content => $composableBuilder( - column: $table.content, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get contentHash => $composableBuilder( - column: $table.contentHash, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get metadata => $composableBuilder( - column: $table.metadata, - builder: (column) => ColumnFilters(column), - ); -} - -class $DocumentOrderingComposer extends Composer<_$TabDatabase, Document> { - $DocumentOrderingComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnOrderings get documentId => $composableBuilder( - column: $table.documentId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get mainDocumentId => $composableBuilder( - column: $table.mainDocumentId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get contextId => $composableBuilder( - column: $table.contextId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get content => $composableBuilder( - column: $table.content, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get contentHash => $composableBuilder( - column: $table.contentHash, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get metadata => $composableBuilder( - column: $table.metadata, - builder: (column) => ColumnOrderings(column), - ); -} - -class $DocumentAnnotationComposer extends Composer<_$TabDatabase, Document> { - $DocumentAnnotationComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - GeneratedColumn get documentId => $composableBuilder( - column: $table.documentId, - builder: (column) => column, - ); - - GeneratedColumn get mainDocumentId => $composableBuilder( - column: $table.mainDocumentId, - builder: (column) => column, - ); - - GeneratedColumn get contextId => - $composableBuilder(column: $table.contextId, builder: (column) => column); - - GeneratedColumn get content => - $composableBuilder(column: $table.content, builder: (column) => column); - - GeneratedColumn get contentHash => $composableBuilder( - column: $table.contentHash, - builder: (column) => column, - ); - - GeneratedColumn get metadata => - $composableBuilder(column: $table.metadata, builder: (column) => column); -} - -class $DocumentTableManager - extends - RootTableManager< - _$TabDatabase, - Document, - DocumentData, - $DocumentFilterComposer, - $DocumentOrderingComposer, - $DocumentAnnotationComposer, - $DocumentCreateCompanionBuilder, - $DocumentUpdateCompanionBuilder, - (DocumentData, BaseReferences<_$TabDatabase, Document, DocumentData>), - DocumentData, - PrefetchHooks Function() - > { - $DocumentTableManager(_$TabDatabase db, Document table) - : super( - TableManagerState( - db: db, - table: table, - createFilteringComposer: - () => $DocumentFilterComposer($db: db, $table: table), - createOrderingComposer: - () => $DocumentOrderingComposer($db: db, $table: table), - createComputedFieldComposer: - () => $DocumentAnnotationComposer($db: db, $table: table), - updateCompanionCallback: - ({ - Value documentId = const Value.absent(), - Value mainDocumentId = const Value.absent(), - Value contextId = const Value.absent(), - Value content = const Value.absent(), - Value contentHash = const Value.absent(), - Value metadata = const Value.absent(), - Value rowid = const Value.absent(), - }) => DocumentCompanion( - documentId: documentId, - mainDocumentId: mainDocumentId, - contextId: contextId, - content: content, - contentHash: contentHash, - metadata: metadata, - rowid: rowid, - ), - createCompanionCallback: - ({ - required String documentId, - Value mainDocumentId = const Value.absent(), - Value contextId = const Value.absent(), - required String content, - required String contentHash, - Value metadata = const Value.absent(), - Value rowid = const Value.absent(), - }) => DocumentCompanion.insert( - documentId: documentId, - mainDocumentId: mainDocumentId, - contextId: contextId, - content: content, - contentHash: contentHash, - metadata: metadata, - rowid: rowid, - ), - withReferenceMapper: - (p0) => - p0 - .map( - (e) => ( - e.readTable(table), - BaseReferences(db, table, e), - ), - ) - .toList(), - prefetchHooksCallback: null, - ), - ); -} - -typedef $DocumentProcessedTableManager = - ProcessedTableManager< - _$TabDatabase, - Document, - DocumentData, - $DocumentFilterComposer, - $DocumentOrderingComposer, - $DocumentAnnotationComposer, - $DocumentCreateCompanionBuilder, - $DocumentUpdateCompanionBuilder, - (DocumentData, BaseReferences<_$TabDatabase, Document, DocumentData>), - DocumentData, - PrefetchHooks Function() - >; typedef $TabFtsCreateCompanionBuilder = TabFtsCompanion Function({ required String title, @@ -3191,254 +2142,6 @@ typedef $TabFtsProcessedTableManager = TabFt, PrefetchHooks Function() >; -typedef $DocumentVecCreateCompanionBuilder = - DocumentVecCompanion Function({ - required String id, - Value mainDocumentId, - Value contextId, - required Uint8List embedding, - required String contentHash, - Value distance, - Value k, - Value rowid, - }); -typedef $DocumentVecUpdateCompanionBuilder = - DocumentVecCompanion Function({ - Value id, - Value mainDocumentId, - Value contextId, - Value embedding, - Value contentHash, - Value distance, - Value k, - Value rowid, - }); - -class $DocumentVecFilterComposer extends Composer<_$TabDatabase, DocumentVec> { - $DocumentVecFilterComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnFilters get id => $composableBuilder( - column: $table.id, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get mainDocumentId => $composableBuilder( - column: $table.mainDocumentId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get contextId => $composableBuilder( - column: $table.contextId, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get embedding => $composableBuilder( - column: $table.embedding, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get contentHash => $composableBuilder( - column: $table.contentHash, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get distance => $composableBuilder( - column: $table.distance, - builder: (column) => ColumnFilters(column), - ); - - ColumnFilters get k => $composableBuilder( - column: $table.k, - builder: (column) => ColumnFilters(column), - ); -} - -class $DocumentVecOrderingComposer - extends Composer<_$TabDatabase, DocumentVec> { - $DocumentVecOrderingComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - ColumnOrderings get id => $composableBuilder( - column: $table.id, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get mainDocumentId => $composableBuilder( - column: $table.mainDocumentId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get contextId => $composableBuilder( - column: $table.contextId, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get embedding => $composableBuilder( - column: $table.embedding, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get contentHash => $composableBuilder( - column: $table.contentHash, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get distance => $composableBuilder( - column: $table.distance, - builder: (column) => ColumnOrderings(column), - ); - - ColumnOrderings get k => $composableBuilder( - column: $table.k, - builder: (column) => ColumnOrderings(column), - ); -} - -class $DocumentVecAnnotationComposer - extends Composer<_$TabDatabase, DocumentVec> { - $DocumentVecAnnotationComposer({ - required super.$db, - required super.$table, - super.joinBuilder, - super.$addJoinBuilderToRootComposer, - super.$removeJoinBuilderFromRootComposer, - }); - GeneratedColumn get id => - $composableBuilder(column: $table.id, builder: (column) => column); - - GeneratedColumn get mainDocumentId => $composableBuilder( - column: $table.mainDocumentId, - builder: (column) => column, - ); - - GeneratedColumn get contextId => - $composableBuilder(column: $table.contextId, builder: (column) => column); - - GeneratedColumn get embedding => - $composableBuilder(column: $table.embedding, builder: (column) => column); - - GeneratedColumn get contentHash => $composableBuilder( - column: $table.contentHash, - builder: (column) => column, - ); - - GeneratedColumn get distance => - $composableBuilder(column: $table.distance, builder: (column) => column); - - GeneratedColumn get k => - $composableBuilder(column: $table.k, builder: (column) => column); -} - -class $DocumentVecTableManager - extends - RootTableManager< - _$TabDatabase, - DocumentVec, - DocumentVecData, - $DocumentVecFilterComposer, - $DocumentVecOrderingComposer, - $DocumentVecAnnotationComposer, - $DocumentVecCreateCompanionBuilder, - $DocumentVecUpdateCompanionBuilder, - ( - DocumentVecData, - BaseReferences<_$TabDatabase, DocumentVec, DocumentVecData>, - ), - DocumentVecData, - PrefetchHooks Function() - > { - $DocumentVecTableManager(_$TabDatabase db, DocumentVec table) - : super( - TableManagerState( - db: db, - table: table, - createFilteringComposer: - () => $DocumentVecFilterComposer($db: db, $table: table), - createOrderingComposer: - () => $DocumentVecOrderingComposer($db: db, $table: table), - createComputedFieldComposer: - () => $DocumentVecAnnotationComposer($db: db, $table: table), - updateCompanionCallback: - ({ - Value id = const Value.absent(), - Value mainDocumentId = const Value.absent(), - Value contextId = const Value.absent(), - Value embedding = const Value.absent(), - Value contentHash = const Value.absent(), - Value distance = const Value.absent(), - Value k = const Value.absent(), - Value rowid = const Value.absent(), - }) => DocumentVecCompanion( - id: id, - mainDocumentId: mainDocumentId, - contextId: contextId, - embedding: embedding, - contentHash: contentHash, - distance: distance, - k: k, - rowid: rowid, - ), - createCompanionCallback: - ({ - required String id, - Value mainDocumentId = const Value.absent(), - Value contextId = const Value.absent(), - required Uint8List embedding, - required String contentHash, - Value distance = const Value.absent(), - Value k = const Value.absent(), - Value rowid = const Value.absent(), - }) => DocumentVecCompanion.insert( - id: id, - mainDocumentId: mainDocumentId, - contextId: contextId, - embedding: embedding, - contentHash: contentHash, - distance: distance, - k: k, - rowid: rowid, - ), - withReferenceMapper: - (p0) => - p0 - .map( - (e) => ( - e.readTable(table), - BaseReferences(db, table, e), - ), - ) - .toList(), - prefetchHooksCallback: null, - ), - ); -} - -typedef $DocumentVecProcessedTableManager = - ProcessedTableManager< - _$TabDatabase, - DocumentVec, - DocumentVecData, - $DocumentVecFilterComposer, - $DocumentVecOrderingComposer, - $DocumentVecAnnotationComposer, - $DocumentVecCreateCompanionBuilder, - $DocumentVecUpdateCompanionBuilder, - ( - DocumentVecData, - BaseReferences<_$TabDatabase, DocumentVec, DocumentVecData>, - ), - DocumentVecData, - PrefetchHooks Function() - >; class $TabDatabaseManager { final _$TabDatabase _db; @@ -3446,9 +2149,5 @@ class $TabDatabaseManager { $ContainerTableManager get container => $ContainerTableManager(_db, _db.container); $TabTableManager get tab => $TabTableManager(_db, _db.tab); - $DocumentTableManager get document => - $DocumentTableManager(_db, _db.document); $TabFtsTableManager get tabFts => $TabFtsTableManager(_db, _db.tabFts); - $DocumentVecTableManager get documentVec => - $DocumentVecTableManager(_db, _db.documentVec); } diff --git a/app/lib/features/geckoview/features/tabs/data/providers.dart b/app/lib/features/geckoview/features/tabs/data/providers.dart index e2adad39..eb2c8f98 100644 --- a/app/lib/features/geckoview/features/tabs/data/providers.dart +++ b/app/lib/features/geckoview/features/tabs/data/providers.dart @@ -6,9 +6,7 @@ import 'package:riverpod/riverpod.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite3_flutter_libs/sqlite3_flutter_libs.dart'; -import 'package:sqlite3_vec/sqlite3_vec.dart'; import 'package:universal_io/io.dart'; -import 'package:weblibre/core/providers/models.dart'; import 'package:weblibre/data/database/functions/lexo_rank_functions.dart'; import 'package:weblibre/features/geckoview/features/tabs/data/database/database.dart'; @@ -16,8 +14,6 @@ part 'providers.g.dart'; @Riverpod(keepAlive: true) TabDatabase tabDatabase(Ref ref) { - final dimensions = ref.watch(embeddingDimensionsProvider); - final db = TabDatabase( LazyDatabase(() async { // put the database file, called db.sqlite here, into the documents folder @@ -37,8 +33,6 @@ TabDatabase tabDatabase(Ref ref) { // Explicitly tell it about the correct temporary directory. sqlite3.tempDirectory = cachebase; - Sqlite3Vec.ensureExtensionLoaded(); - return NativeDatabase.createInBackground( file, setup: (database) { @@ -46,7 +40,6 @@ TabDatabase tabDatabase(Ref ref) { }, ); }), - embeddingDimensions: dimensions, ); ref.onDispose(() async { diff --git a/app/lib/features/geckoview/features/tabs/data/providers.g.dart b/app/lib/features/geckoview/features/tabs/data/providers.g.dart index 701bb61a..04313759 100644 --- a/app/lib/features/geckoview/features/tabs/data/providers.g.dart +++ b/app/lib/features/geckoview/features/tabs/data/providers.g.dart @@ -6,7 +6,7 @@ part of 'providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$tabDatabaseHash() => r'db190f01d7e70e65b9f44403effa82ec503144a0'; +String _$tabDatabaseHash() => r'422bd4789296dc271fabbd5906f2e2ab16bccaa3'; /// See also [tabDatabase]. @ProviderFor(tabDatabase) diff --git a/app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.dart b/app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.dart deleted file mode 100644 index a1ec9d8e..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.dart +++ /dev/null @@ -1,79 +0,0 @@ -import 'dart:async'; - -import 'package:exceptions/exceptions.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/message_types.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/providers.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/repositories/chat_message.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart'; - -part 'chat_backend.g.dart'; - -@Riverpod() -class ChatBackend extends _$ChatBackend { - late ChatMetadata? _metadata; - - late ChatMessageRepository _chatRepository; - - late QAMemoryChain _qaMemoryChain; - - Future? _embeddingsUpdate; - Future prepareEmbeddings() { - if (_embeddingsUpdate != null) { - return _embeddingsUpdate!; - } - - _embeddingsUpdate = ref - .read(documentRepositoryProvider.notifier) - .updateEmbeddings( - mainDocumentId: _metadata?.mainDocumentId, - contextId: _metadata?.contextId, - ); - - _embeddingsUpdate!.whenComplete(() => _embeddingsUpdate = null); - - return _embeddingsUpdate!; - } - - Future> processQAMessage(String input) async { - //TODO: fix result mess - - final humanMessageResult = await _chatRepository.insertTextMessage( - author: MessageAuthor.human, - content: input, - ); - - await prepareEmbeddings(); - - return humanMessageResult.flatMapAsync((humanMessage) async { - await _chatRepository.setTyping(author: MessageAuthor.ai, typing: true); - - final result = await _qaMemoryChain.processQuestion(humanMessage.text); - await _chatRepository.setTyping(author: MessageAuthor.ai, typing: false); - - await _chatRepository.insertTextMessage( - author: MessageAuthor.ai, - content: result.answer, - ); - }); - } - - @override - void build(String chatId) { - _metadata = ref.watch( - chatMetadataProvider(chatId).select((value) => value.valueOrNull), - ); - - _chatRepository = ref.watch(chatMessageRepositoryProvider(chatId).notifier); - - _qaMemoryChain = ref.watch( - qAMemoryChainProvider( - chatId: chatId, - mainDocumentId: _metadata?.mainDocumentId, - contextId: _metadata?.contextId, - ).notifier, - ); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.g.dart b/app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.g.dart deleted file mode 100644 index 6c6fbc38..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/domain/chat_backend.g.dart +++ /dev/null @@ -1,161 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'chat_backend.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$chatBackendHash() => r'befdad9878bf0a1a8a196af5e3006978026eb8ba'; - -/// Copied from Dart SDK -class _SystemHash { - _SystemHash._(); - - static int combine(int hash, int value) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + value); - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10)); - return hash ^ (hash >> 6); - } - - static int finish(int hash) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3)); - // ignore: parameter_assignments - hash = hash ^ (hash >> 11); - return 0x1fffffff & (hash + ((0x00003fff & hash) << 15)); - } -} - -abstract class _$ChatBackend extends BuildlessAutoDisposeNotifier { - late final String chatId; - - void build(String chatId); -} - -/// See also [ChatBackend]. -@ProviderFor(ChatBackend) -const chatBackendProvider = ChatBackendFamily(); - -/// See also [ChatBackend]. -class ChatBackendFamily extends Family { - /// See also [ChatBackend]. - const ChatBackendFamily(); - - /// See also [ChatBackend]. - ChatBackendProvider call(String chatId) { - return ChatBackendProvider(chatId); - } - - @override - ChatBackendProvider getProviderOverride( - covariant ChatBackendProvider provider, - ) { - return call(provider.chatId); - } - - static const Iterable? _dependencies = null; - - @override - Iterable? get dependencies => _dependencies; - - static const Iterable? _allTransitiveDependencies = null; - - @override - Iterable? get allTransitiveDependencies => - _allTransitiveDependencies; - - @override - String? get name => r'chatBackendProvider'; -} - -/// See also [ChatBackend]. -class ChatBackendProvider - extends AutoDisposeNotifierProviderImpl { - /// See also [ChatBackend]. - ChatBackendProvider(String chatId) - : this._internal( - () => ChatBackend()..chatId = chatId, - from: chatBackendProvider, - name: r'chatBackendProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$chatBackendHash, - dependencies: ChatBackendFamily._dependencies, - allTransitiveDependencies: ChatBackendFamily._allTransitiveDependencies, - chatId: chatId, - ); - - ChatBackendProvider._internal( - super._createNotifier, { - required super.name, - required super.dependencies, - required super.allTransitiveDependencies, - required super.debugGetCreateSourceHash, - required super.from, - required this.chatId, - }) : super.internal(); - - final String chatId; - - @override - void runNotifierBuild(covariant ChatBackend notifier) { - return notifier.build(chatId); - } - - @override - Override overrideWith(ChatBackend Function() create) { - return ProviderOverride( - origin: this, - override: ChatBackendProvider._internal( - () => create()..chatId = chatId, - from: from, - name: null, - dependencies: null, - allTransitiveDependencies: null, - debugGetCreateSourceHash: null, - chatId: chatId, - ), - ); - } - - @override - AutoDisposeNotifierProviderElement createElement() { - return _ChatBackendProviderElement(this); - } - - @override - bool operator ==(Object other) { - return other is ChatBackendProvider && other.chatId == chatId; - } - - @override - int get hashCode { - var hash = _SystemHash.combine(0, runtimeType.hashCode); - hash = _SystemHash.combine(hash, chatId.hashCode); - - return _SystemHash.finish(hash); - } -} - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -mixin ChatBackendRef on AutoDisposeNotifierProviderRef { - /// The parameter `chatId` of this provider. - String get chatId; -} - -class _ChatBackendProviderElement - extends AutoDisposeNotifierProviderElement - with ChatBackendRef { - _ChatBackendProviderElement(super.provider); - - @override - String get chatId => (origin as ChatBackendProvider).chatId; -} - -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/chat_text_message.dart b/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/chat_text_message.dart deleted file mode 100644 index bc39cfe0..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/chat_text_message.dart +++ /dev/null @@ -1,57 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:flutter_markdown/flutter_markdown.dart'; -import 'package:provider/provider.dart'; - -class ChatTextMessage extends StatelessWidget { - final TextMessage message; - final int index; - final EdgeInsetsGeometry? padding; - final BorderRadiusGeometry? borderRadius; - final double? onlyEmojiFontSize; - - const ChatTextMessage({ - super.key, - required this.message, - required this.index, - this.padding = const EdgeInsets.symmetric(horizontal: 16, vertical: 10), - this.borderRadius = const BorderRadius.all(Radius.circular(12)), - this.onlyEmojiFontSize = 48, - }); - - @override - Widget build(BuildContext context) { - final textMessageTheme = context.select( - (ChatTheme theme) => theme.textMessageTheme, - ); - final isSentByMe = context.watch() == message.authorId; - final paragraphStyle = - isSentByMe - ? textMessageTheme.sentTextStyle - : textMessageTheme.receivedTextStyle; - - return Container( - padding: padding, - decoration: - message.isOnlyEmoji == true - ? null - : BoxDecoration( - color: - isSentByMe - ? textMessageTheme.sentBackgroundColor - : textMessageTheme.receivedBackgroundColor, - borderRadius: borderRadius, - ), - child: MarkdownBody( - data: message.text, - selectable: true, - styleSheet: MarkdownStyleSheet( - p: - message.isOnlyEmoji == true - ? paragraphStyle?.copyWith(fontSize: onlyEmojiFontSize) - : paragraphStyle, - ), - ), - ); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/qa_chat_input.dart b/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/qa_chat_input.dart deleted file mode 100644 index a873c798..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/qa_chat_input.dart +++ /dev/null @@ -1,186 +0,0 @@ -import 'dart:ui'; - -import 'package:flutter/material.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:flutter_chat_ui/flutter_chat_ui.dart'; -import 'package:provider/provider.dart'; -import 'package:weblibre/presentation/widgets/speech_to_text_button.dart'; - -typedef OnMessageTapCallback = void Function(Message message); -typedef OnMessageSendCallback = void Function(String text); -typedef OnAttachmentTapCallback = VoidCallback; - -class QaChatInput extends StatefulWidget { - final double? left; - final double? right; - final double? top; - final double? bottom; - final double? sigmaX; - final double? sigmaY; - final EdgeInsetsGeometry? padding; - final Widget? attachmentIcon; - final Widget? sendIcon; - final double? gap; - final InputBorder? inputBorder; - final bool? filled; - final Widget? topWidget; - final bool? handleSafeArea; - - const QaChatInput({ - super.key, - this.left = 0, - this.right = 0, - this.top, - this.bottom = 0, - this.sigmaX = 20, - this.sigmaY = 20, - this.padding = const EdgeInsets.all(8.0), - this.attachmentIcon = const Icon(Icons.attachment), - this.sendIcon = const Icon(Icons.send), - this.gap = 8, - this.inputBorder = const OutlineInputBorder( - borderSide: BorderSide.none, - borderRadius: BorderRadius.all(Radius.circular(24)), - ), - this.filled = true, - this.topWidget, - this.handleSafeArea = true, - }); - - @override - State createState() => _QaChatInputState(); -} - -class _QaChatInputState extends State { - final GlobalKey _inputKey = GlobalKey(); - final TextEditingController _textController = TextEditingController(); - - @override - void initState() { - super.initState(); - WidgetsBinding.instance.addPostFrameCallback((_) => _updateInputHeight()); - } - - @override - void didUpdateWidget(covariant QaChatInput oldWidget) { - super.didUpdateWidget(oldWidget); - WidgetsBinding.instance.addPostFrameCallback((_) => _updateInputHeight()); - } - - @override - void dispose() { - _textController.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - final bottomSafeArea = - widget.handleSafeArea == true - ? MediaQuery.of(context).padding.bottom - : 0.0; - final inputTheme = context.select((ChatTheme theme) => theme.inputTheme); - final onAttachmentTap = context.read(); - - return Positioned( - left: widget.left, - right: widget.right, - top: widget.top, - bottom: widget.bottom, - child: ClipRect( - child: BackdropFilter( - filter: ImageFilter.blur( - // TODO: remove backdrop filter if both are 0 - sigmaX: widget.sigmaX ?? 0, - sigmaY: widget.sigmaY ?? 0, - ), - child: Container( - key: _inputKey, - color: inputTheme.backgroundColor, - child: Column( - children: [ - if (widget.topWidget != null) widget.topWidget!, - Padding( - padding: - widget.handleSafeArea == true - ? (widget.padding?.add( - EdgeInsets.only(bottom: bottomSafeArea), - ) ?? - EdgeInsets.only(bottom: bottomSafeArea)) - : (widget.padding ?? EdgeInsets.zero), - child: Row( - children: [ - if (widget.attachmentIcon != null) - IconButton( - icon: widget.attachmentIcon!, - color: inputTheme.hintStyle?.color, - onPressed: onAttachmentTap, - ) - else - const SizedBox.shrink(), - SizedBox(width: widget.gap), - Expanded( - child: TextField( - controller: _textController, - decoration: InputDecoration( - hintText: 'Type a message', - hintStyle: inputTheme.hintStyle, - border: widget.inputBorder, - filled: widget.filled, - fillColor: inputTheme.textFieldColor, - hoverColor: Colors.transparent, - suffixIcon: SpeechToTextButton( - onTextReceived: (data) { - _textController.text = data.toString(); - }, - ), - ), - style: inputTheme.textStyle, - onSubmitted: _handleSubmitted, - textInputAction: TextInputAction.send, - ), - ), - SizedBox(width: widget.gap), - if (widget.sendIcon != null) - IconButton( - icon: widget.sendIcon!, - color: inputTheme.hintStyle?.color, - onPressed: - () => _handleSubmitted(_textController.text), - ) - else - const SizedBox.shrink(), - ], - ), - ), - ], - ), - ), - ), - ), - ); - } - - void _updateInputHeight() { - if (!mounted) return; - - final renderBox = - _inputKey.currentContext?.findRenderObject() as RenderBox?; - if (renderBox != null) { - final height = renderBox.size.height; - final bottomSafeArea = MediaQuery.of(context).padding.bottom; - - context.read().updateHeight( - // only set real height of the input, ignoring safe area - widget.handleSafeArea == true ? height - bottomSafeArea : height, - ); - } - } - - void _handleSubmitted(String text) { - if (text.isNotEmpty) { - context.read()?.call(text); - _textController.clear(); - } - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/tab_qa_chat.dart b/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/tab_qa_chat.dart deleted file mode 100644 index aa54a4db..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/presentation/widgets/tab_qa_chat.dart +++ /dev/null @@ -1,72 +0,0 @@ -import 'package:collection/collection.dart'; -import 'package:flutter/material.dart'; -import 'package:flutter_chat_core/flutter_chat_core.dart'; -import 'package:flutter_chat_ui/flutter_chat_ui.dart'; -import 'package:flutter_hooks/flutter_hooks.dart'; -import 'package:hooks_riverpod/hooks_riverpod.dart'; -import 'package:weblibre/features/chat/features/chat_store/data/models/message_types.dart'; -import 'package:weblibre/features/chat/features/chat_store/domain/providers.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/chat/domain/chat_backend.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/chat/presentation/widgets/chat_text_message.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/chat/presentation/widgets/qa_chat_input.dart'; -import 'package:weblibre/presentation/hooks/on_initialization.dart'; - -class TabQaChat extends HookConsumerWidget { - final String chatId; - final ScrollController? scrollController; - - const TabQaChat({required this.chatId, this.scrollController, super.key}); - - @override - Widget build(BuildContext context, WidgetRef ref) { - final chatBackend = ref.watch(chatBackendProvider(chatId).notifier); - final chatController = ref.watch(chatControllerProvider(chatId)); - - // final crossCache = useMemoized(() => CrossCache()); - final chatScrollController = scrollController ?? useScrollController(); - - useOnInitialization(() async { - await chatBackend.prepareEmbeddings(); - }); - - return Chat( - darkTheme: ChatTheme.dark( - backgroundColor: Theme.of(context).colorScheme.surface, - inputTheme: InputTheme( - backgroundColor: Theme.of(context).colorScheme.surface, - ), - ), - theme: ChatTheme.light( - backgroundColor: Theme.of(context).colorScheme.surface, - ), - builders: Builders( - textMessageBuilder: - (context, message, index) => - ChatTextMessage(message: message, index: index), - customMessageBuilder: - (context, message, index) => Container( - padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 10), - decoration: const BoxDecoration( - color: Color(0xFFF0F0F0), - borderRadius: BorderRadius.all(Radius.circular(12)), - ), - child: const IsTypingIndicator(), - ), - inputBuilder: (context) => const QaChatInput(attachmentIcon: null), - ), - chatController: chatController, - // crossCache: crossCache, - scrollController: chatScrollController, - onMessageSend: (text) async { - await chatBackend.processQAMessage(text); - }, - currentUserId: MessageAuthor.human.user.id, - resolveUser: - (id) => Future.value( - MessageAuthor.values - .firstWhereOrNull((user) => user.user.id == id) - ?.user, - ), - ); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.dart b/app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.dart deleted file mode 100644 index 13cce9e6..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.dart +++ /dev/null @@ -1,59 +0,0 @@ -import 'package:langchain/langchain.dart'; -import 'package:langchain_openai/langchain_openai.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:weblibre/core/providers/models.dart'; -import 'package:weblibre/features/geckoview/features/tabs/data/providers.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/domain/sqlite_vector_store.dart'; - -part 'qa_memory_chain.g.dart'; - -typedef QAResult = ({String answer, List docs}); - -@Riverpod() -class QAMemoryChain extends _$QAMemoryChain { - late RetrievalQAChain _retrievalQA; - - @override - void build({ - required String chatId, - String? mainDocumentId, - String? contextId, - }) { - final chatModel = ref.watch(chatModelProvider); - final embeddingsModel = ref.watch(embeddingModelProvider); - final db = ref.watch(tabDatabaseProvider); - - final retriever = SqliteVectorStore( - db.vectorDao, - embeddings: embeddingsModel, - mainDocumentId: mainDocumentId, - contextId: contextId, - ).asRetriever( - defaultOptions: const VectorStoreRetrieverOptions( - searchType: VectorStoreSimilaritySearch(k: 6), - ), - ); - - final qaChain = OpenAIQAWithSourcesChain(llm: chatModel); - final docPrompt = PromptTemplate.fromTemplate( - 'Content: {page_content}\nSource: {source}', - ); - - final finalQAChain = StuffDocumentsChain( - llmChain: qaChain, - documentPrompt: docPrompt, - ); - - _retrievalQA = RetrievalQAChain( - retriever: retriever, - combineDocumentsChain: finalQAChain, - ); - } - - Future processQuestion(String input) async { - final result = await _retrievalQA(input); - final qaResult = result['result'] as QAWithSources; - - return (answer: qaResult.answer, docs: []); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.g.dart b/app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.g.dart deleted file mode 100644 index 3b9f028d..00000000 --- a/app/lib/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.g.dart +++ /dev/null @@ -1,219 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'qa_memory_chain.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$qAMemoryChainHash() => r'f05ff182416cd8ac8c35f046081652f8964684a8'; - -/// Copied from Dart SDK -class _SystemHash { - _SystemHash._(); - - static int combine(int hash, int value) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + value); - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10)); - return hash ^ (hash >> 6); - } - - static int finish(int hash) { - // ignore: parameter_assignments - hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3)); - // ignore: parameter_assignments - hash = hash ^ (hash >> 11); - return 0x1fffffff & (hash + ((0x00003fff & hash) << 15)); - } -} - -abstract class _$QAMemoryChain extends BuildlessAutoDisposeNotifier { - late final String chatId; - late final String? mainDocumentId; - late final String? contextId; - - void build({ - required String chatId, - String? mainDocumentId, - String? contextId, - }); -} - -/// See also [QAMemoryChain]. -@ProviderFor(QAMemoryChain) -const qAMemoryChainProvider = QAMemoryChainFamily(); - -/// See also [QAMemoryChain]. -class QAMemoryChainFamily extends Family { - /// See also [QAMemoryChain]. - const QAMemoryChainFamily(); - - /// See also [QAMemoryChain]. - QAMemoryChainProvider call({ - required String chatId, - String? mainDocumentId, - String? contextId, - }) { - return QAMemoryChainProvider( - chatId: chatId, - mainDocumentId: mainDocumentId, - contextId: contextId, - ); - } - - @override - QAMemoryChainProvider getProviderOverride( - covariant QAMemoryChainProvider provider, - ) { - return call( - chatId: provider.chatId, - mainDocumentId: provider.mainDocumentId, - contextId: provider.contextId, - ); - } - - static const Iterable? _dependencies = null; - - @override - Iterable? get dependencies => _dependencies; - - static const Iterable? _allTransitiveDependencies = null; - - @override - Iterable? get allTransitiveDependencies => - _allTransitiveDependencies; - - @override - String? get name => r'qAMemoryChainProvider'; -} - -/// See also [QAMemoryChain]. -class QAMemoryChainProvider - extends AutoDisposeNotifierProviderImpl { - /// See also [QAMemoryChain]. - QAMemoryChainProvider({ - required String chatId, - String? mainDocumentId, - String? contextId, - }) : this._internal( - () => - QAMemoryChain() - ..chatId = chatId - ..mainDocumentId = mainDocumentId - ..contextId = contextId, - from: qAMemoryChainProvider, - name: r'qAMemoryChainProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$qAMemoryChainHash, - dependencies: QAMemoryChainFamily._dependencies, - allTransitiveDependencies: - QAMemoryChainFamily._allTransitiveDependencies, - chatId: chatId, - mainDocumentId: mainDocumentId, - contextId: contextId, - ); - - QAMemoryChainProvider._internal( - super._createNotifier, { - required super.name, - required super.dependencies, - required super.allTransitiveDependencies, - required super.debugGetCreateSourceHash, - required super.from, - required this.chatId, - required this.mainDocumentId, - required this.contextId, - }) : super.internal(); - - final String chatId; - final String? mainDocumentId; - final String? contextId; - - @override - void runNotifierBuild(covariant QAMemoryChain notifier) { - return notifier.build( - chatId: chatId, - mainDocumentId: mainDocumentId, - contextId: contextId, - ); - } - - @override - Override overrideWith(QAMemoryChain Function() create) { - return ProviderOverride( - origin: this, - override: QAMemoryChainProvider._internal( - () => - create() - ..chatId = chatId - ..mainDocumentId = mainDocumentId - ..contextId = contextId, - from: from, - name: null, - dependencies: null, - allTransitiveDependencies: null, - debugGetCreateSourceHash: null, - chatId: chatId, - mainDocumentId: mainDocumentId, - contextId: contextId, - ), - ); - } - - @override - AutoDisposeNotifierProviderElement createElement() { - return _QAMemoryChainProviderElement(this); - } - - @override - bool operator ==(Object other) { - return other is QAMemoryChainProvider && - other.chatId == chatId && - other.mainDocumentId == mainDocumentId && - other.contextId == contextId; - } - - @override - int get hashCode { - var hash = _SystemHash.combine(0, runtimeType.hashCode); - hash = _SystemHash.combine(hash, chatId.hashCode); - hash = _SystemHash.combine(hash, mainDocumentId.hashCode); - hash = _SystemHash.combine(hash, contextId.hashCode); - - return _SystemHash.finish(hash); - } -} - -@Deprecated('Will be removed in 3.0. Use Ref instead') -// ignore: unused_element -mixin QAMemoryChainRef on AutoDisposeNotifierProviderRef { - /// The parameter `chatId` of this provider. - String get chatId; - - /// The parameter `mainDocumentId` of this provider. - String? get mainDocumentId; - - /// The parameter `contextId` of this provider. - String? get contextId; -} - -class _QAMemoryChainProviderElement - extends AutoDisposeNotifierProviderElement - with QAMemoryChainRef { - _QAMemoryChainProviderElement(super.provider); - - @override - String get chatId => (origin as QAMemoryChainProvider).chatId; - @override - String? get mainDocumentId => - (origin as QAMemoryChainProvider).mainDocumentId; - @override - String? get contextId => (origin as QAMemoryChainProvider).contextId; -} - -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart deleted file mode 100644 index 3e27d10a..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart +++ /dev/null @@ -1,173 +0,0 @@ -import 'dart:convert'; - -import 'package:drift/drift.dart'; -import 'package:langchain/langchain.dart' as langchain; -import 'package:weblibre/core/uuid.dart'; -import 'package:weblibre/features/geckoview/features/tabs/data/database/database.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/utils/hash.dart'; -import 'package:weblibre/utils/langchain_utils.dart'; - -part 'vector.g.dart'; - -@DriftAccessor() -class VectorDao extends DatabaseAccessor with _$VectorDaoMixin { - VectorDao(super.attachedDatabase); - - Future deleteDocuments(List ids) { - return db.document.deleteWhere((doc) => doc.documentId.isIn(ids)); - } - - Future deleteDocumentsByMainDocumentId(String mainDocumentId) { - return db.document.deleteWhere( - (doc) => doc.mainDocumentId.equals(mainDocumentId), - ); - } - - SingleOrNullSelectable getDocumentById(String documentId) { - return db.document.select() - ..where((row) => row.documentId.equals(documentId)); - } - - Selectable getDocuments({ - String? mainDocumentId, - String? contextId, - }) { - final statement = db.document.select(); - - if (mainDocumentId != null) { - statement.where((row) => row.mainDocumentId.equals(mainDocumentId)); - } - - if (contextId != null) { - statement.where((row) => row.contextId.equals(contextId)); - } - - return statement; - } - - Selectable getDocumentsWithMissingEmbeddings({ - String? mainDocumentId, - String? contextId, - }) { - return db.missingDocumentEmbeddings( - mainDocumentId: mainDocumentId, - contextId: contextId, - ); - } - - Future insertDocuments( - List documents, { - String? mainDocumentId, - String? contextId, - }) { - return db.document.insertAll( - documents.map((document) { - final documentId = document.id ?? uuid.v4(); - final contentHash = sha2(document.pageContent); - - return DocumentCompanion.insert( - documentId: documentId, - mainDocumentId: Value.absentIfNull(mainDocumentId), - contextId: Value.absentIfNull(contextId), - content: document.pageContent, - metadata: - (document.metadata.isNotEmpty) - ? Value(jsonEncode(document.metadata)) - : const Value.absent(), - contentHash: contentHash, - ); - }), - mode: InsertMode.insertOrReplace, - ); - } - - Future insertEmbeddings(List<(DocumentData, List)> documents) { - return db.documentVec.insertAll( - documents.map((doc) { - final (document, vector) = doc; - - return DocumentVecCompanion.insert( - id: document.documentId, - embedding: serializeVector(vector), - mainDocumentId: Value.absentIfNull(document.mainDocumentId), - contextId: Value.absentIfNull(document.contextId), - contentHash: document.contentHash, - ); - }), - ); - } - - Future> insertDocumentsWithEmbedings( - List<(langchain.Document, List)> documents, { - String? mainDocumentId, - String? contextId, - }) async { - final insertedIds = []; - - await db.batch((b) async { - final insertableDocuments = >[]; - final insertableEmbeddings = >[]; - - for (final (document, vector) in documents) { - final documentId = document.id ?? uuid.v4(); - final contentHash = sha2(document.pageContent); - - insertedIds.add(documentId); - - insertableDocuments.add( - DocumentCompanion.insert( - documentId: documentId, - mainDocumentId: Value.absentIfNull(mainDocumentId), - contextId: Value.absentIfNull(contextId), - content: document.pageContent, - metadata: - (document.metadata.isNotEmpty) - ? Value(jsonEncode(document.metadata)) - : const Value.absent(), - contentHash: contentHash, - ), - ); - - insertableEmbeddings.add( - DocumentVecCompanion.insert( - id: documentId, - embedding: serializeVector(vector), - mainDocumentId: Value.absentIfNull(mainDocumentId), - contextId: Value.absentIfNull(contextId), - contentHash: contentHash, - ), - ); - } - - await db.document.insertAll( - insertableDocuments, - mode: InsertMode.insertOrReplace, - ); - - await db.documentVec.insertAll(insertableEmbeddings); - }); - - return insertedIds; - } - - Selectable vectorSearch({ - required langchain.VectorStoreSimilaritySearch config, - required List searchVectors, - String? mainDocumentId, - String? contextId, - }) { - assert(config.filter?.isNotEmpty ?? true, 'Filters are unsupported'); - assert( - !(mainDocumentId != null && contextId != null), - 'Either filter by document or context, not both', - ); - - return db.queryVectors( - searchVectors: serializeVector(searchVectors), - k: config.k, - mainDocumentId: mainDocumentId, - contextId: contextId, - ); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.g.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.g.dart deleted file mode 100644 index ae20acd0..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.g.dart +++ /dev/null @@ -1,6 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'vector.dart'; - -// ignore_for_file: type=lint -mixin _$VectorDaoMixin on DatabaseAccessor {} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/migrator.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/migrator.dart deleted file mode 100644 index 5d85a827..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/migrator.dart +++ /dev/null @@ -1,15 +0,0 @@ -class VectorDatabaseMigrator { - final int dimensions; - - String get vectorTableDefinition => ''' -CREATE VIRTUAL TABLE document_vec using vec0( - id TEXT PRIMARY KEY, - main_document_id TEXT, - context_id TEXT PARTITION KEY, - embedding float[$dimensions], - content_hash TEXT -); -'''; - - VectorDatabaseMigrator({required this.dimensions}); -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/vector_store.drift b/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/vector_store.drift deleted file mode 100644 index 739f1de5..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/data/database/vector_store.drift +++ /dev/null @@ -1,72 +0,0 @@ -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart'; - -CREATE TABLE document( - document_id TEXT NOT NULL PRIMARY KEY, - main_document_id TEXT, - context_id TEXT, - content TEXT NOT NULL, - content_hash TEXT NOT NULL, - metadata TEXT -); - -CREATE INDEX document_main_document_id ON document (main_document_id); -CREATE INDEX document_context_id ON document (context_id); - -CREATE TABLE document_vec( - -- dummy definition of virtual table - -- keep in sync with defined table definition - - id TEXT NOT NULL PRIMARY KEY, - main_document_id TEXT, - context_id TEXT, - embedding BLOB NOT NULL, - content_hash TEXT NOT NULL, - - distance REAL, - k INTEGER -); - -CREATE TRIGGER document_delete AFTER DELETE ON document BEGIN - DELETE FROM document_vec WHERE id = old.document_id; -END; - -CREATE TRIGGER document_update_delete AFTER UPDATE ON document BEGIN - DELETE FROM document_vec WHERE id = new.document_id AND content_hash != new.content_hash; -END; - -missingDocumentEmbeddings( - :main_document_id AS TEXT OR NULL, - :context_id AS TEXT OR NULL -): - SELECT - doc.* - FROM document doc - WHERE - doc.main_document_id IS COALESCE(:main_document_id, doc.main_document_id) AND - doc.context_id IS COALESCE(:context_id, doc.context_id) AND - NOT EXISTS ( - SELECT 1 - FROM document_vec vec - WHERE vec.id = doc.document_id - ); - -queryVectors( - :searchVectors AS BLOB, - :main_document_id AS TEXT OR NULL, - :context_id AS TEXT OR NULL -) WITH VectorResult: - SELECT - vec.id, - doc.main_document_id, - doc.context_id, - doc.content, - doc.metadata, - vec.distance - FROM document_vec vec - INNER JOIN document doc ON doc.document_id = vec.id - WHERE - vec.embedding MATCH :searchVectors AND - vec.k = :k AND - vec.main_document_id IS COALESCE(:main_document_id, vec.main_document_id) AND - vec.context_id IS COALESCE(:context_id, vec.context_id) - ORDER BY vec.distance; \ No newline at end of file diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart deleted file mode 100644 index f71398b9..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart +++ /dev/null @@ -1,40 +0,0 @@ -import 'dart:convert'; - -import 'package:fast_equatable/fast_equatable.dart'; -import 'package:langchain/langchain.dart'; - -class VectorResult with FastEquatable { - final String id; - final String? mainDocumentId; - final String? contextId; - final String content; - final Map metadata; - final double distance; - - VectorResult({ - required this.id, - required this.mainDocumentId, - required this.contextId, - required this.content, - required String? metadata, - required double? distance, - }) : metadata = - (metadata != null) - ? jsonDecode(metadata) as Map - : const {}, - distance = distance!; - - Document toDocument() { - return Document(id: id, pageContent: content, metadata: metadata); - } - - @override - List get hashParameters => [ - id, - mainDocumentId, - contextId, - content, - metadata, - distance, - ]; -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart deleted file mode 100644 index 7bd1ca8e..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart +++ /dev/null @@ -1,115 +0,0 @@ -import 'dart:async'; - -import 'package:collection/collection.dart'; -import 'package:exceptions/exceptions.dart'; -import 'package:langchain/langchain.dart' as langchain; -import 'package:langchain/langchain.dart'; -import 'package:langchain_openai/langchain_openai.dart'; -import 'package:nullability/nullability.dart'; -import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:weblibre/core/providers/models.dart'; -import 'package:weblibre/features/geckoview/domain/providers.dart'; -import 'package:weblibre/features/geckoview/features/tabs/data/providers.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/utils/markdown_document_splitter.dart'; - -part 'document.g.dart'; - -@Riverpod() -class DocumentRepository extends _$DocumentRepository { - late VectorDao _vectorDao; - late OpenAIEmbeddings _embeddings; - - Future _insertMarkdownDocumentsSplitted( - List originalDocuments, { - List<(String, String)> headersToSplitOn = const [ - ('#', 'h1'), - ('##', 'h2'), - ('###', 'h3'), - ('####', 'h4'), - ('#####', 'h5'), - ('######', 'h6'), - ], - int chunkSize = 748, - int chunkOverlap = 150, - }) { - return _vectorDao.transaction(() async { - for (final doc in originalDocuments) { - final splitted = splitMarkdownDocument( - doc, - headersToSplitOn: headersToSplitOn, - chunkSize: chunkSize, - chunkOverlap: chunkOverlap, - ); - - final splittedWithSource = splitted.mapNotNull( - (splitted) => ( - mainDocumentId: splitted.mainDocumentId, - parts: - splitted.parts - .map( - (part) => part.copyWith( - metadata: {...part.metadata, 'source': part.id}, - ), - ) - .toList(), - ), - ); - - await _vectorDao.insertDocuments( - splittedWithSource?.parts ?? [doc], - mainDocumentId: splittedWithSource?.mainDocumentId, - ); - } - }); - } - - Future updateEmbeddings({String? mainDocumentId, String? contextId}) { - return Result.fromAsync(() async { - final missing = - await _vectorDao - .getDocumentsWithMissingEmbeddings( - mainDocumentId: mainDocumentId, - contextId: contextId, - ) - .get(); - - final documentEmbeddings = await _embeddings.embedDocuments( - missing - .map((doc) => langchain.Document(pageContent: doc.content)) - .toList(), - ); - - return _vectorDao.insertEmbeddings( - missing.mapIndexed((i, doc) => (doc, documentEmbeddings[i])).toList(), - ); - }); - } - - @override - void build() { - _vectorDao = ref.watch(tabDatabaseProvider).vectorDao; - _embeddings = ref.watch(embeddingModelProvider); - - final tabContentService = ref.watch(tabContentServiceProvider); - - final tabContentSub = tabContentService.tabContentStream.listen(( - content, - ) async { - final bestContent = - content.isProbablyReaderable - ? content.extractedContentMarkdown - : content.fullContentMarkdown; - - if (bestContent != null) { - await _insertMarkdownDocumentsSplitted([ - Document(id: content.tabId, pageContent: bestContent), - ]); - } - }); - - ref.onDispose(() async { - await tabContentSub.cancel(); - }); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart deleted file mode 100644 index b46e0fa0..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.g.dart +++ /dev/null @@ -1,28 +0,0 @@ -// GENERATED CODE - DO NOT MODIFY BY HAND - -part of 'document.dart'; - -// ************************************************************************** -// RiverpodGenerator -// ************************************************************************** - -String _$documentRepositoryHash() => - r'fd78b5eaf96951724e905b068144dcf2a254dcbb'; - -/// See also [DocumentRepository]. -@ProviderFor(DocumentRepository) -final documentRepositoryProvider = - AutoDisposeNotifierProvider.internal( - DocumentRepository.new, - name: r'documentRepositoryProvider', - debugGetCreateSourceHash: - const bool.fromEnvironment('dart.vm.product') - ? null - : _$documentRepositoryHash, - dependencies: null, - allTransitiveDependencies: null, - ); - -typedef _$DocumentRepository = AutoDisposeNotifier; -// ignore_for_file: type=lint -// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/sqlite_vector_store.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/domain/sqlite_vector_store.dart deleted file mode 100644 index a31c7531..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/domain/sqlite_vector_store.dart +++ /dev/null @@ -1,61 +0,0 @@ -import 'package:collection/collection.dart'; -import 'package:langchain/langchain.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart'; - -class SqliteVectorStore extends VectorStore { - final VectorDao _dao; - - final String? mainDocumentId; - final String? contextId; - - SqliteVectorStore( - this._dao, { - required super.embeddings, - this.mainDocumentId, - this.contextId, - }); - - @override - Future> addVectors({ - required List documents, - required List> vectors, - }) { - return _dao.insertDocumentsWithEmbedings( - documents.mapIndexed((i, doc) => (doc, vectors[i])).toList(), - contextId: contextId, - mainDocumentId: mainDocumentId, - ); - } - - @override - Future delete({required List ids}) { - return _dao.deleteDocuments(ids); - } - - @override - Future> similaritySearchByVectorWithScores({ - required List embedding, - VectorStoreSimilaritySearch config = const VectorStoreSimilaritySearch(), - }) async { - Iterable results = - await _dao - .vectorSearch( - config: config, - searchVectors: embedding, - contextId: contextId, - mainDocumentId: mainDocumentId, - ) - .get(); - - if (config.scoreThreshold != null) { - results = results.where( - (result) => result.distance >= config.scoreThreshold!, - ); - } - - return results - .map((result) => (result.toDocument(), result.distance)) - .toList(); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/utils/document_uuid.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/utils/document_uuid.dart deleted file mode 100644 index 5f813ccd..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/utils/document_uuid.dart +++ /dev/null @@ -1,26 +0,0 @@ -import 'dart:typed_data'; - -import 'package:uuid/data.dart'; -import 'package:uuid/parsing.dart'; -import 'package:uuid/rng.dart'; -import 'package:weblibre/core/uuid.dart'; - -final _rng = CryptoRNG(); - -class DocumentUuid { - final Uint8List _baseBytes; - - DocumentUuid([Uint8List? baseBytes]) - : _baseBytes = baseBytes ?? _rng.generate(); - - factory DocumentUuid.fromUuid(String uuid) { - return DocumentUuid(UuidParsing.parseAsByteList(uuid)); - } - - String getDocumentPartUuid(int sequence) { - final bytes = Uint8List.fromList(_baseBytes); - bytes.buffer.asByteData().setInt16(14, sequence); - - return uuid.v8g(config: V8GenericOptions(bytes)); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/utils/hash.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/utils/hash.dart deleted file mode 100644 index ce0d4878..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/utils/hash.dart +++ /dev/null @@ -1,17 +0,0 @@ -import 'dart:convert'; - -import 'package:crypto/crypto.dart'; -import 'package:flutter/foundation.dart'; - -String sha2(String input) { - return sha256.convert(utf8.encode(input)).toString(); -} - -Future sha2Isolated(String input) async { - if (input.length < 1048576) { - // Less than 1MB - return sha2(input); - } else { - return await compute(sha2, input); - } -} diff --git a/app/lib/features/geckoview/features/tabs/features/vector_store/utils/markdown_document_splitter.dart b/app/lib/features/geckoview/features/tabs/features/vector_store/utils/markdown_document_splitter.dart deleted file mode 100644 index cbc62ade..00000000 --- a/app/lib/features/geckoview/features/tabs/features/vector_store/utils/markdown_document_splitter.dart +++ /dev/null @@ -1,116 +0,0 @@ -import 'package:collection/collection.dart'; -import 'package:langchain/langchain.dart'; -import 'package:weblibre/core/uuid.dart'; -import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/utils/document_uuid.dart'; - -typedef DocumentParts = ({String mainDocumentId, List parts}); - -List _mergeShortDocuments( - List documents, - int maxLength, { - // ignore: unused_element_parameter - String separator = '\n', - required Document Function(Document a, Document b) doMerge, - int Function(String) lengthFunction = TextSplitter.defaultLengthFunction, -}) { - final result = []; - - Document? current; - for (final doc in documents) { - if (current == null) { - current = doc; - } else if (lengthFunction(current.pageContent) + - lengthFunction(doc.pageContent) <= - maxLength) { - current = doMerge(current, doc); - } else { - result.add(current); - current = doc; - } - } - - if (current != null) { - result.add(current); - } - - return result; -} - -List _headerValueList(dynamic value) { - return switch (value) { - String _ => [value], - List _ => value, - _ => throw Exception('Unsupported type'), - }; -} - -DocumentParts? splitMarkdownDocument( - Document originalDoc, { - required List<(String, String)> headersToSplitOn, - required int chunkSize, - required int chunkOverlap, -}) { - final markdownHeaderSplitter = MarkdownHeaderTextSplitter( - stripHeaders: false, - headersToSplitOn: headersToSplitOn, - ); - - final markdownTextSplitter = MarkdownTextSplitter( - chunkSize: chunkSize, - chunkOverlap: chunkOverlap, - ); - - final headerChunks = markdownHeaderSplitter.splitText( - originalDoc.pageContent, - ); - final docChunks = markdownTextSplitter.splitDocuments(headerChunks); - - if (docChunks.length == 1) { - return null; - } - - final headerKeys = headersToSplitOn.map((header) => header.$2).toSet(); - final mergedDocChunks = _mergeShortDocuments( - docChunks, - chunkSize, - doMerge: (a, b) { - //Disallow merging into headers of the same - final intersectingHeaders = a.metadata.keys - .toSet() - .intersection(b.metadata.keys.toSet()) - .intersection(headerKeys); - - return Document( - id: a.id ?? b.id, - pageContent: '${a.pageContent}\n${b.pageContent}', - metadata: { - ...a.metadata, - ...b.metadata, - for (final header in intersectingHeaders) - header: - { - ..._headerValueList(a.metadata[header]), - ..._headerValueList(b.metadata[header]), - }.toList(), - }, - ); - }, - ); - - final mainDocumentId = originalDoc.id ?? uuid.v4(); - final documentIdFactory = DocumentUuid.fromUuid(mainDocumentId); - - return ( - mainDocumentId: mainDocumentId, - parts: - mergedDocChunks - .mapIndexed( - (i, doc) => Document( - id: documentIdFactory.getDocumentPartUuid(i), - pageContent: doc.pageContent, - metadata: mergeMaps(originalDoc.metadata, doc.metadata), - ), - ) - .toList(), - ); -} diff --git a/app/lib/utils/langchain_utils.dart b/app/lib/utils/langchain_utils.dart deleted file mode 100644 index 3562ea7e..00000000 --- a/app/lib/utils/langchain_utils.dart +++ /dev/null @@ -1,12 +0,0 @@ -import 'dart:typed_data'; - -import 'package:langchain/langchain.dart'; - -String combineDocuments(List documents, {String separator = '\n\n'}) { - return documents.map((d) => d.pageContent).join(separator); -} - -Uint8List serializeVector(List vector) { - final buffer = Float32List.fromList(vector).buffer; - return Uint8List.view(buffer); -} diff --git a/app/pubspec.yaml b/app/pubspec.yaml index 2c8f6217..b402aacf 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -10,7 +10,6 @@ dependencies: background_fetch: ^1.3.8 collection: ^1.19.1 copy_with_extension: ^6.0.1 - crypto: ^3.0.6 drift: ^2.26.1 dynamic_color: ^1.7.0 exceptions: ^0.6.1 @@ -20,10 +19,6 @@ dependencies: sdk: flutter flutter_auto_size_text: ^4.1.0 flutter_background_service: ^5.1.0 - flutter_chat_core: - path: ../../flutter_chat_ui/packages/flutter_chat_core - flutter_chat_ui: - path: ../../flutter_chat_ui/packages/flutter_chat_ui flutter_hooks: ^0.21.2 flutter_markdown: ^0.7.7+1 flutter_material_design_icons: ^1.1.7447 @@ -42,16 +37,6 @@ dependencies: http: ^1.4.0 intl: ^0.19.0 json_annotation: ^4.9.0 - langchain: - git: - url: https://github.com/FaFre/langchain_dart.git - ref: static_objectbox - path: packages/langchain - langchain_openai: - git: - url: https://github.com/FaFre/langchain_dart.git - ref: static_objectbox - path: packages/langchain_openai lexo_rank: git: url: https://github.com/FaFre/lexo_rank.git @@ -64,7 +49,6 @@ dependencies: path: ^1.9.1 path_provider: ^2.1.5 pocketbase: ^0.22.0 - provider: ^6.1.5 riverpod: ^2.6.1 riverpod_annotation: ^2.6.1 rss_dart: ^1.0.12 @@ -77,10 +61,7 @@ dependencies: speech_to_text_google_dialog: git: url: https://github.com/FaFre/speech_to_text_google_dialog.git - sqlite3: ^2.7.5 - sqlite3_flutter_libs: ^0.5.32 - sqlite3_vec: - git: https://github.com/FaFre/sqlite3_vec.git + sqlite3_flutter_libs: ^0.5.33 text_scroll: ^0.2.0 timeago: ^3.7.1 tor: @@ -94,22 +75,6 @@ dependencies: url_launcher: ^6.3.1 uuid: ^4.5.1 -dependency_overrides: - cross_cache: - path: ../../flutter_chat_ui/packages/cross_cache - flutter_chat_core: - path: ../../flutter_chat_ui/packages/flutter_chat_core - langchain_core: - git: - url: https://github.com/FaFre/langchain_dart.git - ref: static_objectbox - path: packages/langchain_core - tavily_dart: - git: - url: https://github.com/FaFre/langchain_dart.git - ref: static_objectbox - path: packages/tavily_dart - dev_dependencies: build_runner: ^2.4.15 copy_with_extension_gen: ^6.0.1 diff --git a/app/pubspec_overrides.yaml b/app/pubspec_overrides.yaml index 5a8fe41b..d1cf0013 100644 --- a/app/pubspec_overrides.yaml +++ b/app/pubspec_overrides.yaml @@ -1,20 +1,6 @@ -# melos_managed_dependency_overrides: flutter_mozilla_components,simple_intent_receiver,cross_cache,flutter_chat_core,langchain_core,tavily_dart +# melos_managed_dependency_overrides: flutter_mozilla_components,simple_intent_receiver dependency_overrides: flutter_mozilla_components: path: ../packages/flutter_mozilla_components simple_intent_receiver: path: ../packages/simple_intent_receiver - cross_cache: - path: ../../flutter_chat_ui/packages/cross_cache - flutter_chat_core: - path: ../../flutter_chat_ui/packages/flutter_chat_core - langchain_core: - git: - url: https://github.com/FaFre/langchain_dart.git - ref: static_objectbox - path: packages/langchain_core - tavily_dart: - git: - url: https://github.com/FaFre/langchain_dart.git - ref: static_objectbox - path: packages/tavily_dart