strip chat
This commit is contained in:
Fabian Freund
@@ -10,13 +10,8 @@ import 'package:share_plus/share_plus.dart';
|
||||
import 'package:weblibre/data/models/web_page_info.dart';
|
||||
import 'package:weblibre/features/bangs/domain/providers/bangs.dart';
|
||||
import 'package:weblibre/features/bangs/presentation/widgets/site_search.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/domain/repositories/chat_metadata.dart';
|
||||
import 'package:weblibre/features/geckoview/domain/controllers/bottom_sheet.dart';
|
||||
import 'package:weblibre/features/geckoview/domain/providers/tab_session.dart';
|
||||
import 'package:weblibre/features/geckoview/domain/providers/tab_state.dart';
|
||||
import 'package:weblibre/features/geckoview/domain/repositories/tab.dart';
|
||||
import 'package:weblibre/features/geckoview/features/browser/domain/entities/sheet.dart';
|
||||
import 'package:weblibre/features/geckoview/features/browser/presentation/widgets/browser_modules/address_with_suggestions_field.dart';
|
||||
import 'package:weblibre/features/user/domain/providers.dart';
|
||||
import 'package:weblibre/presentation/widgets/failure_widget.dart';
|
||||
@@ -236,59 +231,6 @@ class WebPageDialog extends HookConsumerWidget {
|
||||
}
|
||||
},
|
||||
),
|
||||
const Divider(),
|
||||
ListTile(
|
||||
leading: const Icon(MdiIcons.brain),
|
||||
title: const Text('QA Chat'),
|
||||
onTap: () async {
|
||||
final selectedTabId =
|
||||
ref.read(selectedTabStateProvider)?.id;
|
||||
|
||||
if (selectedTabId != null) {
|
||||
final updateResult = await ref
|
||||
.read(
|
||||
chatMetadataRepositoryProvider(
|
||||
selectedTabId,
|
||||
).notifier,
|
||||
)
|
||||
.updateMetadata(
|
||||
ChatMetadata(mainDocumentId: selectedTabId),
|
||||
);
|
||||
|
||||
updateResult.onSuccess((_) {
|
||||
ref
|
||||
.read(
|
||||
bottomSheetControllerProvider.notifier,
|
||||
)
|
||||
.show(
|
||||
TabQaChatSheet(chatId: selectedTabId),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
if (context.mounted) {
|
||||
context.pop();
|
||||
}
|
||||
},
|
||||
),
|
||||
// ListTile(
|
||||
// leading: Icon(MdiIcons.text),
|
||||
// title: const Text('Summarize'),
|
||||
// onTap: () async {
|
||||
// final summarizerUrl = uri_builder.summarizerUri(
|
||||
// document: SharedUrl(url),
|
||||
// mode: SummarizerMode.keyMoments,
|
||||
// );
|
||||
|
||||
// await ref
|
||||
// .read(tabRepositoryProvider.notifier)
|
||||
// .addTab(url: summarizerUrl);
|
||||
|
||||
// if (context.mounted) {
|
||||
// context.pop();
|
||||
// }
|
||||
// },
|
||||
// ),
|
||||
],
|
||||
),
|
||||
);
|
||||
|
||||
@@ -24,7 +24,6 @@ import 'package:weblibre/features/geckoview/features/browser/presentation/widget
|
||||
import 'package:weblibre/features/geckoview/features/contextmenu/extensions/hit_result.dart';
|
||||
import 'package:weblibre/features/geckoview/features/find_in_page/presentation/widgets/find_in_page.dart';
|
||||
import 'package:weblibre/features/geckoview/features/readerview/presentation/widgets/reader_appearance_button.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/chat/presentation/widgets/tab_qa_chat.dart';
|
||||
import 'package:weblibre/utils/ui_helper.dart' as ui_helper;
|
||||
|
||||
class BrowserScreen extends HookConsumerWidget {
|
||||
@@ -371,12 +370,6 @@ class _QaSheet extends HookConsumerWidget {
|
||||
),
|
||||
),
|
||||
),
|
||||
Expanded(
|
||||
child: TabQaChat(
|
||||
chatId: chatId,
|
||||
scrollController: scrollController,
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
);
|
||||
|
||||
-2
@@ -19,7 +19,6 @@ import 'package:weblibre/features/geckoview/features/browser/domain/providers/li
|
||||
import 'package:weblibre/features/geckoview/features/browser/domain/services/browser_data.dart';
|
||||
import 'package:weblibre/features/geckoview/features/browser/domain/services/engine_settings_replication.dart';
|
||||
import 'package:weblibre/features/geckoview/features/browser/domain/services/proxy_settings_replication.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart';
|
||||
import 'package:weblibre/features/user/domain/repositories/cache.dart';
|
||||
import 'package:weblibre/features/user/domain/repositories/general_settings.dart';
|
||||
import 'package:weblibre/features/user/domain/services/local_authentication.dart';
|
||||
@@ -154,7 +153,6 @@ class _BrowserViewState extends ConsumerState<BrowserView>
|
||||
|
||||
//Initialize and register dependencies
|
||||
ref.listenManual(tabRepositoryProvider, (previous, next) {});
|
||||
ref.listenManual(documentRepositoryProvider, (previous, next) {});
|
||||
|
||||
ref.listenManual(selectionActionServiceProvider, (previous, next) {});
|
||||
|
||||
|
||||
@@ -7,20 +7,12 @@ import 'package:weblibre/features/geckoview/features/tabs/data/database/daos/con
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/database/daos/tab.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/models/container_data.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/models/tab_query_result.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/migrator.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart';
|
||||
import 'package:weblibre/features/search/domain/fts_tokenizer.dart';
|
||||
|
||||
part 'database.g.dart';
|
||||
|
||||
@DriftDatabase(
|
||||
include: {'database.drift'},
|
||||
daos: [ContainerDao, TabDao, VectorDao],
|
||||
)
|
||||
@DriftDatabase(include: {'database.drift'}, daos: [ContainerDao, TabDao])
|
||||
class TabDatabase extends _$TabDatabase with TrigramQueryBuilderMixin {
|
||||
final int embeddingDimensions;
|
||||
|
||||
@override
|
||||
final int schemaVersion = 2;
|
||||
|
||||
@@ -31,23 +23,11 @@ class TabDatabase extends _$TabDatabase with TrigramQueryBuilderMixin {
|
||||
|
||||
@override
|
||||
MigrationStrategy get migration => MigrationStrategy(
|
||||
onCreate: (m) async {
|
||||
final migrator = VectorDatabaseMigrator(dimensions: embeddingDimensions);
|
||||
|
||||
await m.database.customStatement(migrator.vectorTableDefinition);
|
||||
|
||||
//instead of m.createAll(); we igoner vec0 table
|
||||
for (final entity in allSchemaEntities.where(
|
||||
(entity) => entity is! DocumentVec,
|
||||
)) {
|
||||
await m.create(entity);
|
||||
}
|
||||
},
|
||||
beforeOpen: (details) async {
|
||||
await customStatement('PRAGMA foreign_keys = ON;');
|
||||
await optimizeFtsIndex();
|
||||
},
|
||||
);
|
||||
|
||||
TabDatabase(super.e, {required this.embeddingDimensions});
|
||||
TabDatabase(super.e);
|
||||
}
|
||||
|
||||
@@ -25,14 +25,6 @@ CREATE TABLE tab (
|
||||
timestamp DATETIME NOT NULL
|
||||
);
|
||||
|
||||
import '../../features/vector_store/data/database/vector_store.drift';
|
||||
|
||||
-- automativcally remove documents and embeddings on delete
|
||||
-- isnert and updates are managed code side
|
||||
CREATE TRIGGER tab_document_delete AFTER DELETE ON tab BEGIN
|
||||
DELETE FROM document WHERE main_document_id = old.id;
|
||||
END;
|
||||
|
||||
CREATE VIRTUAL TABLE tab_fts
|
||||
USING fts5(
|
||||
title,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -6,9 +6,7 @@ import 'package:riverpod/riverpod.dart';
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:sqlite3/sqlite3.dart';
|
||||
import 'package:sqlite3_flutter_libs/sqlite3_flutter_libs.dart';
|
||||
import 'package:sqlite3_vec/sqlite3_vec.dart';
|
||||
import 'package:universal_io/io.dart';
|
||||
import 'package:weblibre/core/providers/models.dart';
|
||||
import 'package:weblibre/data/database/functions/lexo_rank_functions.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/database/database.dart';
|
||||
|
||||
@@ -16,8 +14,6 @@ part 'providers.g.dart';
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
TabDatabase tabDatabase(Ref ref) {
|
||||
final dimensions = ref.watch(embeddingDimensionsProvider);
|
||||
|
||||
final db = TabDatabase(
|
||||
LazyDatabase(() async {
|
||||
// put the database file, called db.sqlite here, into the documents folder
|
||||
@@ -37,8 +33,6 @@ TabDatabase tabDatabase(Ref ref) {
|
||||
// Explicitly tell it about the correct temporary directory.
|
||||
sqlite3.tempDirectory = cachebase;
|
||||
|
||||
Sqlite3Vec.ensureExtensionLoaded();
|
||||
|
||||
return NativeDatabase.createInBackground(
|
||||
file,
|
||||
setup: (database) {
|
||||
@@ -46,7 +40,6 @@ TabDatabase tabDatabase(Ref ref) {
|
||||
},
|
||||
);
|
||||
}),
|
||||
embeddingDimensions: dimensions,
|
||||
);
|
||||
|
||||
ref.onDispose(() async {
|
||||
|
||||
@@ -6,7 +6,7 @@ part of 'providers.dart';
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
String _$tabDatabaseHash() => r'db190f01d7e70e65b9f44403effa82ec503144a0';
|
||||
String _$tabDatabaseHash() => r'422bd4789296dc271fabbd5906f2e2ab16bccaa3';
|
||||
|
||||
/// See also [tabDatabase].
|
||||
@ProviderFor(tabDatabase)
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:exceptions/exceptions.dart';
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/data/models/chat_metadata.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/data/models/message_types.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/domain/providers.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/domain/repositories/chat_message.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/chat/services/qa_memory_chain.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/domain/repositories/document.dart';
|
||||
|
||||
part 'chat_backend.g.dart';
|
||||
|
||||
@Riverpod()
|
||||
class ChatBackend extends _$ChatBackend {
|
||||
late ChatMetadata? _metadata;
|
||||
|
||||
late ChatMessageRepository _chatRepository;
|
||||
|
||||
late QAMemoryChain _qaMemoryChain;
|
||||
|
||||
Future<void>? _embeddingsUpdate;
|
||||
Future<void> prepareEmbeddings() {
|
||||
if (_embeddingsUpdate != null) {
|
||||
return _embeddingsUpdate!;
|
||||
}
|
||||
|
||||
_embeddingsUpdate = ref
|
||||
.read(documentRepositoryProvider.notifier)
|
||||
.updateEmbeddings(
|
||||
mainDocumentId: _metadata?.mainDocumentId,
|
||||
contextId: _metadata?.contextId,
|
||||
);
|
||||
|
||||
_embeddingsUpdate!.whenComplete(() => _embeddingsUpdate = null);
|
||||
|
||||
return _embeddingsUpdate!;
|
||||
}
|
||||
|
||||
Future<Result<void>> processQAMessage(String input) async {
|
||||
//TODO: fix result mess
|
||||
|
||||
final humanMessageResult = await _chatRepository.insertTextMessage(
|
||||
author: MessageAuthor.human,
|
||||
content: input,
|
||||
);
|
||||
|
||||
await prepareEmbeddings();
|
||||
|
||||
return humanMessageResult.flatMapAsync((humanMessage) async {
|
||||
await _chatRepository.setTyping(author: MessageAuthor.ai, typing: true);
|
||||
|
||||
final result = await _qaMemoryChain.processQuestion(humanMessage.text);
|
||||
await _chatRepository.setTyping(author: MessageAuthor.ai, typing: false);
|
||||
|
||||
await _chatRepository.insertTextMessage(
|
||||
author: MessageAuthor.ai,
|
||||
content: result.answer,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
void build(String chatId) {
|
||||
_metadata = ref.watch(
|
||||
chatMetadataProvider(chatId).select((value) => value.valueOrNull),
|
||||
);
|
||||
|
||||
_chatRepository = ref.watch(chatMessageRepositoryProvider(chatId).notifier);
|
||||
|
||||
_qaMemoryChain = ref.watch(
|
||||
qAMemoryChainProvider(
|
||||
chatId: chatId,
|
||||
mainDocumentId: _metadata?.mainDocumentId,
|
||||
contextId: _metadata?.contextId,
|
||||
).notifier,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'chat_backend.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
String _$chatBackendHash() => r'befdad9878bf0a1a8a196af5e3006978026eb8ba';
|
||||
|
||||
/// Copied from Dart SDK
|
||||
class _SystemHash {
|
||||
_SystemHash._();
|
||||
|
||||
static int combine(int hash, int value) {
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + value);
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10));
|
||||
return hash ^ (hash >> 6);
|
||||
}
|
||||
|
||||
static int finish(int hash) {
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3));
|
||||
// ignore: parameter_assignments
|
||||
hash = hash ^ (hash >> 11);
|
||||
return 0x1fffffff & (hash + ((0x00003fff & hash) << 15));
|
||||
}
|
||||
}
|
||||
|
||||
abstract class _$ChatBackend extends BuildlessAutoDisposeNotifier<void> {
|
||||
late final String chatId;
|
||||
|
||||
void build(String chatId);
|
||||
}
|
||||
|
||||
/// See also [ChatBackend].
|
||||
@ProviderFor(ChatBackend)
|
||||
const chatBackendProvider = ChatBackendFamily();
|
||||
|
||||
/// See also [ChatBackend].
|
||||
class ChatBackendFamily extends Family<void> {
|
||||
/// See also [ChatBackend].
|
||||
const ChatBackendFamily();
|
||||
|
||||
/// See also [ChatBackend].
|
||||
ChatBackendProvider call(String chatId) {
|
||||
return ChatBackendProvider(chatId);
|
||||
}
|
||||
|
||||
@override
|
||||
ChatBackendProvider getProviderOverride(
|
||||
covariant ChatBackendProvider provider,
|
||||
) {
|
||||
return call(provider.chatId);
|
||||
}
|
||||
|
||||
static const Iterable<ProviderOrFamily>? _dependencies = null;
|
||||
|
||||
@override
|
||||
Iterable<ProviderOrFamily>? get dependencies => _dependencies;
|
||||
|
||||
static const Iterable<ProviderOrFamily>? _allTransitiveDependencies = null;
|
||||
|
||||
@override
|
||||
Iterable<ProviderOrFamily>? get allTransitiveDependencies =>
|
||||
_allTransitiveDependencies;
|
||||
|
||||
@override
|
||||
String? get name => r'chatBackendProvider';
|
||||
}
|
||||
|
||||
/// See also [ChatBackend].
|
||||
class ChatBackendProvider
|
||||
extends AutoDisposeNotifierProviderImpl<ChatBackend, void> {
|
||||
/// See also [ChatBackend].
|
||||
ChatBackendProvider(String chatId)
|
||||
: this._internal(
|
||||
() => ChatBackend()..chatId = chatId,
|
||||
from: chatBackendProvider,
|
||||
name: r'chatBackendProvider',
|
||||
debugGetCreateSourceHash:
|
||||
const bool.fromEnvironment('dart.vm.product')
|
||||
? null
|
||||
: _$chatBackendHash,
|
||||
dependencies: ChatBackendFamily._dependencies,
|
||||
allTransitiveDependencies: ChatBackendFamily._allTransitiveDependencies,
|
||||
chatId: chatId,
|
||||
);
|
||||
|
||||
ChatBackendProvider._internal(
|
||||
super._createNotifier, {
|
||||
required super.name,
|
||||
required super.dependencies,
|
||||
required super.allTransitiveDependencies,
|
||||
required super.debugGetCreateSourceHash,
|
||||
required super.from,
|
||||
required this.chatId,
|
||||
}) : super.internal();
|
||||
|
||||
final String chatId;
|
||||
|
||||
@override
|
||||
void runNotifierBuild(covariant ChatBackend notifier) {
|
||||
return notifier.build(chatId);
|
||||
}
|
||||
|
||||
@override
|
||||
Override overrideWith(ChatBackend Function() create) {
|
||||
return ProviderOverride(
|
||||
origin: this,
|
||||
override: ChatBackendProvider._internal(
|
||||
() => create()..chatId = chatId,
|
||||
from: from,
|
||||
name: null,
|
||||
dependencies: null,
|
||||
allTransitiveDependencies: null,
|
||||
debugGetCreateSourceHash: null,
|
||||
chatId: chatId,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
AutoDisposeNotifierProviderElement<ChatBackend, void> createElement() {
|
||||
return _ChatBackendProviderElement(this);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is ChatBackendProvider && other.chatId == chatId;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
var hash = _SystemHash.combine(0, runtimeType.hashCode);
|
||||
hash = _SystemHash.combine(hash, chatId.hashCode);
|
||||
|
||||
return _SystemHash.finish(hash);
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated('Will be removed in 3.0. Use Ref instead')
|
||||
// ignore: unused_element
|
||||
mixin ChatBackendRef on AutoDisposeNotifierProviderRef<void> {
|
||||
/// The parameter `chatId` of this provider.
|
||||
String get chatId;
|
||||
}
|
||||
|
||||
class _ChatBackendProviderElement
|
||||
extends AutoDisposeNotifierProviderElement<ChatBackend, void>
|
||||
with ChatBackendRef {
|
||||
_ChatBackendProviderElement(super.provider);
|
||||
|
||||
@override
|
||||
String get chatId => (origin as ChatBackendProvider).chatId;
|
||||
}
|
||||
|
||||
// ignore_for_file: type=lint
|
||||
// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package
|
||||
-57
@@ -1,57 +0,0 @@
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_chat_core/flutter_chat_core.dart';
|
||||
import 'package:flutter_markdown/flutter_markdown.dart';
|
||||
import 'package:provider/provider.dart';
|
||||
|
||||
class ChatTextMessage extends StatelessWidget {
|
||||
final TextMessage message;
|
||||
final int index;
|
||||
final EdgeInsetsGeometry? padding;
|
||||
final BorderRadiusGeometry? borderRadius;
|
||||
final double? onlyEmojiFontSize;
|
||||
|
||||
const ChatTextMessage({
|
||||
super.key,
|
||||
required this.message,
|
||||
required this.index,
|
||||
this.padding = const EdgeInsets.symmetric(horizontal: 16, vertical: 10),
|
||||
this.borderRadius = const BorderRadius.all(Radius.circular(12)),
|
||||
this.onlyEmojiFontSize = 48,
|
||||
});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final textMessageTheme = context.select(
|
||||
(ChatTheme theme) => theme.textMessageTheme,
|
||||
);
|
||||
final isSentByMe = context.watch<String>() == message.authorId;
|
||||
final paragraphStyle =
|
||||
isSentByMe
|
||||
? textMessageTheme.sentTextStyle
|
||||
: textMessageTheme.receivedTextStyle;
|
||||
|
||||
return Container(
|
||||
padding: padding,
|
||||
decoration:
|
||||
message.isOnlyEmoji == true
|
||||
? null
|
||||
: BoxDecoration(
|
||||
color:
|
||||
isSentByMe
|
||||
? textMessageTheme.sentBackgroundColor
|
||||
: textMessageTheme.receivedBackgroundColor,
|
||||
borderRadius: borderRadius,
|
||||
),
|
||||
child: MarkdownBody(
|
||||
data: message.text,
|
||||
selectable: true,
|
||||
styleSheet: MarkdownStyleSheet(
|
||||
p:
|
||||
message.isOnlyEmoji == true
|
||||
? paragraphStyle?.copyWith(fontSize: onlyEmojiFontSize)
|
||||
: paragraphStyle,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
-186
@@ -1,186 +0,0 @@
|
||||
import 'dart:ui';
|
||||
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_chat_core/flutter_chat_core.dart';
|
||||
import 'package:flutter_chat_ui/flutter_chat_ui.dart';
|
||||
import 'package:provider/provider.dart';
|
||||
import 'package:weblibre/presentation/widgets/speech_to_text_button.dart';
|
||||
|
||||
typedef OnMessageTapCallback = void Function(Message message);
|
||||
typedef OnMessageSendCallback = void Function(String text);
|
||||
typedef OnAttachmentTapCallback = VoidCallback;
|
||||
|
||||
class QaChatInput extends StatefulWidget {
|
||||
final double? left;
|
||||
final double? right;
|
||||
final double? top;
|
||||
final double? bottom;
|
||||
final double? sigmaX;
|
||||
final double? sigmaY;
|
||||
final EdgeInsetsGeometry? padding;
|
||||
final Widget? attachmentIcon;
|
||||
final Widget? sendIcon;
|
||||
final double? gap;
|
||||
final InputBorder? inputBorder;
|
||||
final bool? filled;
|
||||
final Widget? topWidget;
|
||||
final bool? handleSafeArea;
|
||||
|
||||
const QaChatInput({
|
||||
super.key,
|
||||
this.left = 0,
|
||||
this.right = 0,
|
||||
this.top,
|
||||
this.bottom = 0,
|
||||
this.sigmaX = 20,
|
||||
this.sigmaY = 20,
|
||||
this.padding = const EdgeInsets.all(8.0),
|
||||
this.attachmentIcon = const Icon(Icons.attachment),
|
||||
this.sendIcon = const Icon(Icons.send),
|
||||
this.gap = 8,
|
||||
this.inputBorder = const OutlineInputBorder(
|
||||
borderSide: BorderSide.none,
|
||||
borderRadius: BorderRadius.all(Radius.circular(24)),
|
||||
),
|
||||
this.filled = true,
|
||||
this.topWidget,
|
||||
this.handleSafeArea = true,
|
||||
});
|
||||
|
||||
@override
|
||||
State<QaChatInput> createState() => _QaChatInputState();
|
||||
}
|
||||
|
||||
class _QaChatInputState extends State<QaChatInput> {
|
||||
final GlobalKey _inputKey = GlobalKey();
|
||||
final TextEditingController _textController = TextEditingController();
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
super.initState();
|
||||
WidgetsBinding.instance.addPostFrameCallback((_) => _updateInputHeight());
|
||||
}
|
||||
|
||||
@override
|
||||
void didUpdateWidget(covariant QaChatInput oldWidget) {
|
||||
super.didUpdateWidget(oldWidget);
|
||||
WidgetsBinding.instance.addPostFrameCallback((_) => _updateInputHeight());
|
||||
}
|
||||
|
||||
@override
|
||||
void dispose() {
|
||||
_textController.dispose();
|
||||
super.dispose();
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
final bottomSafeArea =
|
||||
widget.handleSafeArea == true
|
||||
? MediaQuery.of(context).padding.bottom
|
||||
: 0.0;
|
||||
final inputTheme = context.select((ChatTheme theme) => theme.inputTheme);
|
||||
final onAttachmentTap = context.read<OnAttachmentTapCallback?>();
|
||||
|
||||
return Positioned(
|
||||
left: widget.left,
|
||||
right: widget.right,
|
||||
top: widget.top,
|
||||
bottom: widget.bottom,
|
||||
child: ClipRect(
|
||||
child: BackdropFilter(
|
||||
filter: ImageFilter.blur(
|
||||
// TODO: remove backdrop filter if both are 0
|
||||
sigmaX: widget.sigmaX ?? 0,
|
||||
sigmaY: widget.sigmaY ?? 0,
|
||||
),
|
||||
child: Container(
|
||||
key: _inputKey,
|
||||
color: inputTheme.backgroundColor,
|
||||
child: Column(
|
||||
children: [
|
||||
if (widget.topWidget != null) widget.topWidget!,
|
||||
Padding(
|
||||
padding:
|
||||
widget.handleSafeArea == true
|
||||
? (widget.padding?.add(
|
||||
EdgeInsets.only(bottom: bottomSafeArea),
|
||||
) ??
|
||||
EdgeInsets.only(bottom: bottomSafeArea))
|
||||
: (widget.padding ?? EdgeInsets.zero),
|
||||
child: Row(
|
||||
children: [
|
||||
if (widget.attachmentIcon != null)
|
||||
IconButton(
|
||||
icon: widget.attachmentIcon!,
|
||||
color: inputTheme.hintStyle?.color,
|
||||
onPressed: onAttachmentTap,
|
||||
)
|
||||
else
|
||||
const SizedBox.shrink(),
|
||||
SizedBox(width: widget.gap),
|
||||
Expanded(
|
||||
child: TextField(
|
||||
controller: _textController,
|
||||
decoration: InputDecoration(
|
||||
hintText: 'Type a message',
|
||||
hintStyle: inputTheme.hintStyle,
|
||||
border: widget.inputBorder,
|
||||
filled: widget.filled,
|
||||
fillColor: inputTheme.textFieldColor,
|
||||
hoverColor: Colors.transparent,
|
||||
suffixIcon: SpeechToTextButton(
|
||||
onTextReceived: (data) {
|
||||
_textController.text = data.toString();
|
||||
},
|
||||
),
|
||||
),
|
||||
style: inputTheme.textStyle,
|
||||
onSubmitted: _handleSubmitted,
|
||||
textInputAction: TextInputAction.send,
|
||||
),
|
||||
),
|
||||
SizedBox(width: widget.gap),
|
||||
if (widget.sendIcon != null)
|
||||
IconButton(
|
||||
icon: widget.sendIcon!,
|
||||
color: inputTheme.hintStyle?.color,
|
||||
onPressed:
|
||||
() => _handleSubmitted(_textController.text),
|
||||
)
|
||||
else
|
||||
const SizedBox.shrink(),
|
||||
],
|
||||
),
|
||||
),
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
void _updateInputHeight() {
|
||||
if (!mounted) return;
|
||||
|
||||
final renderBox =
|
||||
_inputKey.currentContext?.findRenderObject() as RenderBox?;
|
||||
if (renderBox != null) {
|
||||
final height = renderBox.size.height;
|
||||
final bottomSafeArea = MediaQuery.of(context).padding.bottom;
|
||||
|
||||
context.read<ChatInputHeightNotifier>().updateHeight(
|
||||
// only set real height of the input, ignoring safe area
|
||||
widget.handleSafeArea == true ? height - bottomSafeArea : height,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void _handleSubmitted(String text) {
|
||||
if (text.isNotEmpty) {
|
||||
context.read<OnMessageSendCallback?>()?.call(text);
|
||||
_textController.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter_chat_core/flutter_chat_core.dart';
|
||||
import 'package:flutter_chat_ui/flutter_chat_ui.dart';
|
||||
import 'package:flutter_hooks/flutter_hooks.dart';
|
||||
import 'package:hooks_riverpod/hooks_riverpod.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/data/models/message_types.dart';
|
||||
import 'package:weblibre/features/chat/features/chat_store/domain/providers.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/chat/domain/chat_backend.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/chat/presentation/widgets/chat_text_message.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/chat/presentation/widgets/qa_chat_input.dart';
|
||||
import 'package:weblibre/presentation/hooks/on_initialization.dart';
|
||||
|
||||
class TabQaChat extends HookConsumerWidget {
|
||||
final String chatId;
|
||||
final ScrollController? scrollController;
|
||||
|
||||
const TabQaChat({required this.chatId, this.scrollController, super.key});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context, WidgetRef ref) {
|
||||
final chatBackend = ref.watch(chatBackendProvider(chatId).notifier);
|
||||
final chatController = ref.watch(chatControllerProvider(chatId));
|
||||
|
||||
// final crossCache = useMemoized(() => CrossCache());
|
||||
final chatScrollController = scrollController ?? useScrollController();
|
||||
|
||||
useOnInitialization(() async {
|
||||
await chatBackend.prepareEmbeddings();
|
||||
});
|
||||
|
||||
return Chat(
|
||||
darkTheme: ChatTheme.dark(
|
||||
backgroundColor: Theme.of(context).colorScheme.surface,
|
||||
inputTheme: InputTheme(
|
||||
backgroundColor: Theme.of(context).colorScheme.surface,
|
||||
),
|
||||
),
|
||||
theme: ChatTheme.light(
|
||||
backgroundColor: Theme.of(context).colorScheme.surface,
|
||||
),
|
||||
builders: Builders(
|
||||
textMessageBuilder:
|
||||
(context, message, index) =>
|
||||
ChatTextMessage(message: message, index: index),
|
||||
customMessageBuilder:
|
||||
(context, message, index) => Container(
|
||||
padding: const EdgeInsets.symmetric(horizontal: 16, vertical: 10),
|
||||
decoration: const BoxDecoration(
|
||||
color: Color(0xFFF0F0F0),
|
||||
borderRadius: BorderRadius.all(Radius.circular(12)),
|
||||
),
|
||||
child: const IsTypingIndicator(),
|
||||
),
|
||||
inputBuilder: (context) => const QaChatInput(attachmentIcon: null),
|
||||
),
|
||||
chatController: chatController,
|
||||
// crossCache: crossCache,
|
||||
scrollController: chatScrollController,
|
||||
onMessageSend: (text) async {
|
||||
await chatBackend.processQAMessage(text);
|
||||
},
|
||||
currentUserId: MessageAuthor.human.user.id,
|
||||
resolveUser:
|
||||
(id) => Future.value(
|
||||
MessageAuthor.values
|
||||
.firstWhereOrNull((user) => user.user.id == id)
|
||||
?.user,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
import 'package:langchain/langchain.dart';
|
||||
import 'package:langchain_openai/langchain_openai.dart';
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:weblibre/core/providers/models.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/providers.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/domain/sqlite_vector_store.dart';
|
||||
|
||||
part 'qa_memory_chain.g.dart';
|
||||
|
||||
typedef QAResult = ({String answer, List<Document> docs});
|
||||
|
||||
@Riverpod()
|
||||
class QAMemoryChain extends _$QAMemoryChain {
|
||||
late RetrievalQAChain _retrievalQA;
|
||||
|
||||
@override
|
||||
void build({
|
||||
required String chatId,
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) {
|
||||
final chatModel = ref.watch(chatModelProvider);
|
||||
final embeddingsModel = ref.watch(embeddingModelProvider);
|
||||
final db = ref.watch(tabDatabaseProvider);
|
||||
|
||||
final retriever = SqliteVectorStore(
|
||||
db.vectorDao,
|
||||
embeddings: embeddingsModel,
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
).asRetriever(
|
||||
defaultOptions: const VectorStoreRetrieverOptions(
|
||||
searchType: VectorStoreSimilaritySearch(k: 6),
|
||||
),
|
||||
);
|
||||
|
||||
final qaChain = OpenAIQAWithSourcesChain(llm: chatModel);
|
||||
final docPrompt = PromptTemplate.fromTemplate(
|
||||
'Content: {page_content}\nSource: {source}',
|
||||
);
|
||||
|
||||
final finalQAChain = StuffDocumentsChain(
|
||||
llmChain: qaChain,
|
||||
documentPrompt: docPrompt,
|
||||
);
|
||||
|
||||
_retrievalQA = RetrievalQAChain(
|
||||
retriever: retriever,
|
||||
combineDocumentsChain: finalQAChain,
|
||||
);
|
||||
}
|
||||
|
||||
Future<QAResult> processQuestion(String input) async {
|
||||
final result = await _retrievalQA(input);
|
||||
final qaResult = result['result'] as QAWithSources;
|
||||
|
||||
return (answer: qaResult.answer, docs: <Document>[]);
|
||||
}
|
||||
}
|
||||
-219
@@ -1,219 +0,0 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'qa_memory_chain.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
String _$qAMemoryChainHash() => r'f05ff182416cd8ac8c35f046081652f8964684a8';
|
||||
|
||||
/// Copied from Dart SDK
|
||||
class _SystemHash {
|
||||
_SystemHash._();
|
||||
|
||||
static int combine(int hash, int value) {
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + value);
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10));
|
||||
return hash ^ (hash >> 6);
|
||||
}
|
||||
|
||||
static int finish(int hash) {
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3));
|
||||
// ignore: parameter_assignments
|
||||
hash = hash ^ (hash >> 11);
|
||||
return 0x1fffffff & (hash + ((0x00003fff & hash) << 15));
|
||||
}
|
||||
}
|
||||
|
||||
abstract class _$QAMemoryChain extends BuildlessAutoDisposeNotifier<void> {
|
||||
late final String chatId;
|
||||
late final String? mainDocumentId;
|
||||
late final String? contextId;
|
||||
|
||||
void build({
|
||||
required String chatId,
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
});
|
||||
}
|
||||
|
||||
/// See also [QAMemoryChain].
|
||||
@ProviderFor(QAMemoryChain)
|
||||
const qAMemoryChainProvider = QAMemoryChainFamily();
|
||||
|
||||
/// See also [QAMemoryChain].
|
||||
class QAMemoryChainFamily extends Family<void> {
|
||||
/// See also [QAMemoryChain].
|
||||
const QAMemoryChainFamily();
|
||||
|
||||
/// See also [QAMemoryChain].
|
||||
QAMemoryChainProvider call({
|
||||
required String chatId,
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) {
|
||||
return QAMemoryChainProvider(
|
||||
chatId: chatId,
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
QAMemoryChainProvider getProviderOverride(
|
||||
covariant QAMemoryChainProvider provider,
|
||||
) {
|
||||
return call(
|
||||
chatId: provider.chatId,
|
||||
mainDocumentId: provider.mainDocumentId,
|
||||
contextId: provider.contextId,
|
||||
);
|
||||
}
|
||||
|
||||
static const Iterable<ProviderOrFamily>? _dependencies = null;
|
||||
|
||||
@override
|
||||
Iterable<ProviderOrFamily>? get dependencies => _dependencies;
|
||||
|
||||
static const Iterable<ProviderOrFamily>? _allTransitiveDependencies = null;
|
||||
|
||||
@override
|
||||
Iterable<ProviderOrFamily>? get allTransitiveDependencies =>
|
||||
_allTransitiveDependencies;
|
||||
|
||||
@override
|
||||
String? get name => r'qAMemoryChainProvider';
|
||||
}
|
||||
|
||||
/// See also [QAMemoryChain].
|
||||
class QAMemoryChainProvider
|
||||
extends AutoDisposeNotifierProviderImpl<QAMemoryChain, void> {
|
||||
/// See also [QAMemoryChain].
|
||||
QAMemoryChainProvider({
|
||||
required String chatId,
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) : this._internal(
|
||||
() =>
|
||||
QAMemoryChain()
|
||||
..chatId = chatId
|
||||
..mainDocumentId = mainDocumentId
|
||||
..contextId = contextId,
|
||||
from: qAMemoryChainProvider,
|
||||
name: r'qAMemoryChainProvider',
|
||||
debugGetCreateSourceHash:
|
||||
const bool.fromEnvironment('dart.vm.product')
|
||||
? null
|
||||
: _$qAMemoryChainHash,
|
||||
dependencies: QAMemoryChainFamily._dependencies,
|
||||
allTransitiveDependencies:
|
||||
QAMemoryChainFamily._allTransitiveDependencies,
|
||||
chatId: chatId,
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
);
|
||||
|
||||
QAMemoryChainProvider._internal(
|
||||
super._createNotifier, {
|
||||
required super.name,
|
||||
required super.dependencies,
|
||||
required super.allTransitiveDependencies,
|
||||
required super.debugGetCreateSourceHash,
|
||||
required super.from,
|
||||
required this.chatId,
|
||||
required this.mainDocumentId,
|
||||
required this.contextId,
|
||||
}) : super.internal();
|
||||
|
||||
final String chatId;
|
||||
final String? mainDocumentId;
|
||||
final String? contextId;
|
||||
|
||||
@override
|
||||
void runNotifierBuild(covariant QAMemoryChain notifier) {
|
||||
return notifier.build(
|
||||
chatId: chatId,
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Override overrideWith(QAMemoryChain Function() create) {
|
||||
return ProviderOverride(
|
||||
origin: this,
|
||||
override: QAMemoryChainProvider._internal(
|
||||
() =>
|
||||
create()
|
||||
..chatId = chatId
|
||||
..mainDocumentId = mainDocumentId
|
||||
..contextId = contextId,
|
||||
from: from,
|
||||
name: null,
|
||||
dependencies: null,
|
||||
allTransitiveDependencies: null,
|
||||
debugGetCreateSourceHash: null,
|
||||
chatId: chatId,
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
AutoDisposeNotifierProviderElement<QAMemoryChain, void> createElement() {
|
||||
return _QAMemoryChainProviderElement(this);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is QAMemoryChainProvider &&
|
||||
other.chatId == chatId &&
|
||||
other.mainDocumentId == mainDocumentId &&
|
||||
other.contextId == contextId;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
var hash = _SystemHash.combine(0, runtimeType.hashCode);
|
||||
hash = _SystemHash.combine(hash, chatId.hashCode);
|
||||
hash = _SystemHash.combine(hash, mainDocumentId.hashCode);
|
||||
hash = _SystemHash.combine(hash, contextId.hashCode);
|
||||
|
||||
return _SystemHash.finish(hash);
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated('Will be removed in 3.0. Use Ref instead')
|
||||
// ignore: unused_element
|
||||
mixin QAMemoryChainRef on AutoDisposeNotifierProviderRef<void> {
|
||||
/// The parameter `chatId` of this provider.
|
||||
String get chatId;
|
||||
|
||||
/// The parameter `mainDocumentId` of this provider.
|
||||
String? get mainDocumentId;
|
||||
|
||||
/// The parameter `contextId` of this provider.
|
||||
String? get contextId;
|
||||
}
|
||||
|
||||
class _QAMemoryChainProviderElement
|
||||
extends AutoDisposeNotifierProviderElement<QAMemoryChain, void>
|
||||
with QAMemoryChainRef {
|
||||
_QAMemoryChainProviderElement(super.provider);
|
||||
|
||||
@override
|
||||
String get chatId => (origin as QAMemoryChainProvider).chatId;
|
||||
@override
|
||||
String? get mainDocumentId =>
|
||||
(origin as QAMemoryChainProvider).mainDocumentId;
|
||||
@override
|
||||
String? get contextId => (origin as QAMemoryChainProvider).contextId;
|
||||
}
|
||||
|
||||
// ignore_for_file: type=lint
|
||||
// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package
|
||||
-173
@@ -1,173 +0,0 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:langchain/langchain.dart' as langchain;
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/database/database.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/utils/hash.dart';
|
||||
import 'package:weblibre/utils/langchain_utils.dart';
|
||||
|
||||
part 'vector.g.dart';
|
||||
|
||||
@DriftAccessor()
|
||||
class VectorDao extends DatabaseAccessor<TabDatabase> with _$VectorDaoMixin {
|
||||
VectorDao(super.attachedDatabase);
|
||||
|
||||
Future<int> deleteDocuments(List<String> ids) {
|
||||
return db.document.deleteWhere((doc) => doc.documentId.isIn(ids));
|
||||
}
|
||||
|
||||
Future<int> deleteDocumentsByMainDocumentId(String mainDocumentId) {
|
||||
return db.document.deleteWhere(
|
||||
(doc) => doc.mainDocumentId.equals(mainDocumentId),
|
||||
);
|
||||
}
|
||||
|
||||
SingleOrNullSelectable<DocumentData> getDocumentById(String documentId) {
|
||||
return db.document.select()
|
||||
..where((row) => row.documentId.equals(documentId));
|
||||
}
|
||||
|
||||
Selectable<DocumentData> getDocuments({
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) {
|
||||
final statement = db.document.select();
|
||||
|
||||
if (mainDocumentId != null) {
|
||||
statement.where((row) => row.mainDocumentId.equals(mainDocumentId));
|
||||
}
|
||||
|
||||
if (contextId != null) {
|
||||
statement.where((row) => row.contextId.equals(contextId));
|
||||
}
|
||||
|
||||
return statement;
|
||||
}
|
||||
|
||||
Selectable<DocumentData> getDocumentsWithMissingEmbeddings({
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) {
|
||||
return db.missingDocumentEmbeddings(
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> insertDocuments(
|
||||
List<langchain.Document> documents, {
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) {
|
||||
return db.document.insertAll(
|
||||
documents.map((document) {
|
||||
final documentId = document.id ?? uuid.v4();
|
||||
final contentHash = sha2(document.pageContent);
|
||||
|
||||
return DocumentCompanion.insert(
|
||||
documentId: documentId,
|
||||
mainDocumentId: Value.absentIfNull(mainDocumentId),
|
||||
contextId: Value.absentIfNull(contextId),
|
||||
content: document.pageContent,
|
||||
metadata:
|
||||
(document.metadata.isNotEmpty)
|
||||
? Value(jsonEncode(document.metadata))
|
||||
: const Value.absent(),
|
||||
contentHash: contentHash,
|
||||
);
|
||||
}),
|
||||
mode: InsertMode.insertOrReplace,
|
||||
);
|
||||
}
|
||||
|
||||
Future<void> insertEmbeddings(List<(DocumentData, List<double>)> documents) {
|
||||
return db.documentVec.insertAll(
|
||||
documents.map((doc) {
|
||||
final (document, vector) = doc;
|
||||
|
||||
return DocumentVecCompanion.insert(
|
||||
id: document.documentId,
|
||||
embedding: serializeVector(vector),
|
||||
mainDocumentId: Value.absentIfNull(document.mainDocumentId),
|
||||
contextId: Value.absentIfNull(document.contextId),
|
||||
contentHash: document.contentHash,
|
||||
);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
Future<List<String>> insertDocumentsWithEmbedings(
|
||||
List<(langchain.Document, List<double>)> documents, {
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) async {
|
||||
final insertedIds = <String>[];
|
||||
|
||||
await db.batch((b) async {
|
||||
final insertableDocuments = <Insertable<DocumentData>>[];
|
||||
final insertableEmbeddings = <Insertable<DocumentVecData>>[];
|
||||
|
||||
for (final (document, vector) in documents) {
|
||||
final documentId = document.id ?? uuid.v4();
|
||||
final contentHash = sha2(document.pageContent);
|
||||
|
||||
insertedIds.add(documentId);
|
||||
|
||||
insertableDocuments.add(
|
||||
DocumentCompanion.insert(
|
||||
documentId: documentId,
|
||||
mainDocumentId: Value.absentIfNull(mainDocumentId),
|
||||
contextId: Value.absentIfNull(contextId),
|
||||
content: document.pageContent,
|
||||
metadata:
|
||||
(document.metadata.isNotEmpty)
|
||||
? Value(jsonEncode(document.metadata))
|
||||
: const Value.absent(),
|
||||
contentHash: contentHash,
|
||||
),
|
||||
);
|
||||
|
||||
insertableEmbeddings.add(
|
||||
DocumentVecCompanion.insert(
|
||||
id: documentId,
|
||||
embedding: serializeVector(vector),
|
||||
mainDocumentId: Value.absentIfNull(mainDocumentId),
|
||||
contextId: Value.absentIfNull(contextId),
|
||||
contentHash: contentHash,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
await db.document.insertAll(
|
||||
insertableDocuments,
|
||||
mode: InsertMode.insertOrReplace,
|
||||
);
|
||||
|
||||
await db.documentVec.insertAll(insertableEmbeddings);
|
||||
});
|
||||
|
||||
return insertedIds;
|
||||
}
|
||||
|
||||
Selectable<VectorResult> vectorSearch({
|
||||
required langchain.VectorStoreSimilaritySearch config,
|
||||
required List<double> searchVectors,
|
||||
String? mainDocumentId,
|
||||
String? contextId,
|
||||
}) {
|
||||
assert(config.filter?.isNotEmpty ?? true, 'Filters are unsupported');
|
||||
assert(
|
||||
!(mainDocumentId != null && contextId != null),
|
||||
'Either filter by document or context, not both',
|
||||
);
|
||||
|
||||
return db.queryVectors(
|
||||
searchVectors: serializeVector(searchVectors),
|
||||
k: config.k,
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
);
|
||||
}
|
||||
}
|
||||
-6
@@ -1,6 +0,0 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'vector.dart';
|
||||
|
||||
// ignore_for_file: type=lint
|
||||
mixin _$VectorDaoMixin on DatabaseAccessor<TabDatabase> {}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
class VectorDatabaseMigrator {
|
||||
final int dimensions;
|
||||
|
||||
String get vectorTableDefinition => '''
|
||||
CREATE VIRTUAL TABLE document_vec using vec0(
|
||||
id TEXT PRIMARY KEY,
|
||||
main_document_id TEXT,
|
||||
context_id TEXT PARTITION KEY,
|
||||
embedding float[$dimensions],
|
||||
content_hash TEXT
|
||||
);
|
||||
''';
|
||||
|
||||
VectorDatabaseMigrator({required this.dimensions});
|
||||
}
|
||||
-72
@@ -1,72 +0,0 @@
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart';
|
||||
|
||||
CREATE TABLE document(
|
||||
document_id TEXT NOT NULL PRIMARY KEY,
|
||||
main_document_id TEXT,
|
||||
context_id TEXT,
|
||||
content TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
metadata TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX document_main_document_id ON document (main_document_id);
|
||||
CREATE INDEX document_context_id ON document (context_id);
|
||||
|
||||
CREATE TABLE document_vec(
|
||||
-- dummy definition of virtual table
|
||||
-- keep in sync with defined table definition
|
||||
|
||||
id TEXT NOT NULL PRIMARY KEY,
|
||||
main_document_id TEXT,
|
||||
context_id TEXT,
|
||||
embedding BLOB NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
|
||||
distance REAL,
|
||||
k INTEGER
|
||||
);
|
||||
|
||||
CREATE TRIGGER document_delete AFTER DELETE ON document BEGIN
|
||||
DELETE FROM document_vec WHERE id = old.document_id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER document_update_delete AFTER UPDATE ON document BEGIN
|
||||
DELETE FROM document_vec WHERE id = new.document_id AND content_hash != new.content_hash;
|
||||
END;
|
||||
|
||||
missingDocumentEmbeddings(
|
||||
:main_document_id AS TEXT OR NULL,
|
||||
:context_id AS TEXT OR NULL
|
||||
):
|
||||
SELECT
|
||||
doc.*
|
||||
FROM document doc
|
||||
WHERE
|
||||
doc.main_document_id IS COALESCE(:main_document_id, doc.main_document_id) AND
|
||||
doc.context_id IS COALESCE(:context_id, doc.context_id) AND
|
||||
NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM document_vec vec
|
||||
WHERE vec.id = doc.document_id
|
||||
);
|
||||
|
||||
queryVectors(
|
||||
:searchVectors AS BLOB,
|
||||
:main_document_id AS TEXT OR NULL,
|
||||
:context_id AS TEXT OR NULL
|
||||
) WITH VectorResult:
|
||||
SELECT
|
||||
vec.id,
|
||||
doc.main_document_id,
|
||||
doc.context_id,
|
||||
doc.content,
|
||||
doc.metadata,
|
||||
vec.distance
|
||||
FROM document_vec vec
|
||||
INNER JOIN document doc ON doc.document_id = vec.id
|
||||
WHERE
|
||||
vec.embedding MATCH :searchVectors AND
|
||||
vec.k = :k AND
|
||||
vec.main_document_id IS COALESCE(:main_document_id, vec.main_document_id) AND
|
||||
vec.context_id IS COALESCE(:context_id, vec.context_id)
|
||||
ORDER BY vec.distance;
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:fast_equatable/fast_equatable.dart';
|
||||
import 'package:langchain/langchain.dart';
|
||||
|
||||
class VectorResult with FastEquatable {
|
||||
final String id;
|
||||
final String? mainDocumentId;
|
||||
final String? contextId;
|
||||
final String content;
|
||||
final Map<String, dynamic> metadata;
|
||||
final double distance;
|
||||
|
||||
VectorResult({
|
||||
required this.id,
|
||||
required this.mainDocumentId,
|
||||
required this.contextId,
|
||||
required this.content,
|
||||
required String? metadata,
|
||||
required double? distance,
|
||||
}) : metadata =
|
||||
(metadata != null)
|
||||
? jsonDecode(metadata) as Map<String, dynamic>
|
||||
: const {},
|
||||
distance = distance!;
|
||||
|
||||
Document toDocument() {
|
||||
return Document(id: id, pageContent: content, metadata: metadata);
|
||||
}
|
||||
|
||||
@override
|
||||
List<Object?> get hashParameters => [
|
||||
id,
|
||||
mainDocumentId,
|
||||
contextId,
|
||||
content,
|
||||
metadata,
|
||||
distance,
|
||||
];
|
||||
}
|
||||
-115
@@ -1,115 +0,0 @@
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:exceptions/exceptions.dart';
|
||||
import 'package:langchain/langchain.dart' as langchain;
|
||||
import 'package:langchain/langchain.dart';
|
||||
import 'package:langchain_openai/langchain_openai.dart';
|
||||
import 'package:nullability/nullability.dart';
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:weblibre/core/providers/models.dart';
|
||||
import 'package:weblibre/features/geckoview/domain/providers.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/data/providers.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/utils/markdown_document_splitter.dart';
|
||||
|
||||
part 'document.g.dart';
|
||||
|
||||
@Riverpod()
|
||||
class DocumentRepository extends _$DocumentRepository {
|
||||
late VectorDao _vectorDao;
|
||||
late OpenAIEmbeddings _embeddings;
|
||||
|
||||
Future<void> _insertMarkdownDocumentsSplitted(
|
||||
List<langchain.Document> originalDocuments, {
|
||||
List<(String, String)> headersToSplitOn = const [
|
||||
('#', 'h1'),
|
||||
('##', 'h2'),
|
||||
('###', 'h3'),
|
||||
('####', 'h4'),
|
||||
('#####', 'h5'),
|
||||
('######', 'h6'),
|
||||
],
|
||||
int chunkSize = 748,
|
||||
int chunkOverlap = 150,
|
||||
}) {
|
||||
return _vectorDao.transaction(() async {
|
||||
for (final doc in originalDocuments) {
|
||||
final splitted = splitMarkdownDocument(
|
||||
doc,
|
||||
headersToSplitOn: headersToSplitOn,
|
||||
chunkSize: chunkSize,
|
||||
chunkOverlap: chunkOverlap,
|
||||
);
|
||||
|
||||
final splittedWithSource = splitted.mapNotNull(
|
||||
(splitted) => (
|
||||
mainDocumentId: splitted.mainDocumentId,
|
||||
parts:
|
||||
splitted.parts
|
||||
.map(
|
||||
(part) => part.copyWith(
|
||||
metadata: {...part.metadata, 'source': part.id},
|
||||
),
|
||||
)
|
||||
.toList(),
|
||||
),
|
||||
);
|
||||
|
||||
await _vectorDao.insertDocuments(
|
||||
splittedWithSource?.parts ?? [doc],
|
||||
mainDocumentId: splittedWithSource?.mainDocumentId,
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Future<Result> updateEmbeddings({String? mainDocumentId, String? contextId}) {
|
||||
return Result.fromAsync(() async {
|
||||
final missing =
|
||||
await _vectorDao
|
||||
.getDocumentsWithMissingEmbeddings(
|
||||
mainDocumentId: mainDocumentId,
|
||||
contextId: contextId,
|
||||
)
|
||||
.get();
|
||||
|
||||
final documentEmbeddings = await _embeddings.embedDocuments(
|
||||
missing
|
||||
.map((doc) => langchain.Document(pageContent: doc.content))
|
||||
.toList(),
|
||||
);
|
||||
|
||||
return _vectorDao.insertEmbeddings(
|
||||
missing.mapIndexed((i, doc) => (doc, documentEmbeddings[i])).toList(),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
void build() {
|
||||
_vectorDao = ref.watch(tabDatabaseProvider).vectorDao;
|
||||
_embeddings = ref.watch(embeddingModelProvider);
|
||||
|
||||
final tabContentService = ref.watch(tabContentServiceProvider);
|
||||
|
||||
final tabContentSub = tabContentService.tabContentStream.listen((
|
||||
content,
|
||||
) async {
|
||||
final bestContent =
|
||||
content.isProbablyReaderable
|
||||
? content.extractedContentMarkdown
|
||||
: content.fullContentMarkdown;
|
||||
|
||||
if (bestContent != null) {
|
||||
await _insertMarkdownDocumentsSplitted([
|
||||
Document(id: content.tabId, pageContent: bestContent),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
ref.onDispose(() async {
|
||||
await tabContentSub.cancel();
|
||||
});
|
||||
}
|
||||
}
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'document.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
String _$documentRepositoryHash() =>
|
||||
r'fd78b5eaf96951724e905b068144dcf2a254dcbb';
|
||||
|
||||
/// See also [DocumentRepository].
|
||||
@ProviderFor(DocumentRepository)
|
||||
final documentRepositoryProvider =
|
||||
AutoDisposeNotifierProvider<DocumentRepository, void>.internal(
|
||||
DocumentRepository.new,
|
||||
name: r'documentRepositoryProvider',
|
||||
debugGetCreateSourceHash:
|
||||
const bool.fromEnvironment('dart.vm.product')
|
||||
? null
|
||||
: _$documentRepositoryHash,
|
||||
dependencies: null,
|
||||
allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
typedef _$DocumentRepository = AutoDisposeNotifier<void>;
|
||||
// ignore_for_file: type=lint
|
||||
// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package
|
||||
-61
@@ -1,61 +0,0 @@
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:langchain/langchain.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/database/daos/vector.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/data/models/vector_result.dart';
|
||||
|
||||
class SqliteVectorStore extends VectorStore {
|
||||
final VectorDao _dao;
|
||||
|
||||
final String? mainDocumentId;
|
||||
final String? contextId;
|
||||
|
||||
SqliteVectorStore(
|
||||
this._dao, {
|
||||
required super.embeddings,
|
||||
this.mainDocumentId,
|
||||
this.contextId,
|
||||
});
|
||||
|
||||
@override
|
||||
Future<List<String>> addVectors({
|
||||
required List<Document> documents,
|
||||
required List<List<double>> vectors,
|
||||
}) {
|
||||
return _dao.insertDocumentsWithEmbedings(
|
||||
documents.mapIndexed((i, doc) => (doc, vectors[i])).toList(),
|
||||
contextId: contextId,
|
||||
mainDocumentId: mainDocumentId,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> delete({required List<String> ids}) {
|
||||
return _dao.deleteDocuments(ids);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<List<(Document, double)>> similaritySearchByVectorWithScores({
|
||||
required List<double> embedding,
|
||||
VectorStoreSimilaritySearch config = const VectorStoreSimilaritySearch(),
|
||||
}) async {
|
||||
Iterable<VectorResult> results =
|
||||
await _dao
|
||||
.vectorSearch(
|
||||
config: config,
|
||||
searchVectors: embedding,
|
||||
contextId: contextId,
|
||||
mainDocumentId: mainDocumentId,
|
||||
)
|
||||
.get();
|
||||
|
||||
if (config.scoreThreshold != null) {
|
||||
results = results.where(
|
||||
(result) => result.distance >= config.scoreThreshold!,
|
||||
);
|
||||
}
|
||||
|
||||
return results
|
||||
.map((result) => (result.toDocument(), result.distance))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:uuid/data.dart';
|
||||
import 'package:uuid/parsing.dart';
|
||||
import 'package:uuid/rng.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
|
||||
final _rng = CryptoRNG();
|
||||
|
||||
class DocumentUuid {
|
||||
final Uint8List _baseBytes;
|
||||
|
||||
DocumentUuid([Uint8List? baseBytes])
|
||||
: _baseBytes = baseBytes ?? _rng.generate();
|
||||
|
||||
factory DocumentUuid.fromUuid(String uuid) {
|
||||
return DocumentUuid(UuidParsing.parseAsByteList(uuid));
|
||||
}
|
||||
|
||||
String getDocumentPartUuid(int sequence) {
|
||||
final bytes = Uint8List.fromList(_baseBytes);
|
||||
bytes.buffer.asByteData().setInt16(14, sequence);
|
||||
|
||||
return uuid.v8g(config: V8GenericOptions(bytes));
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import 'dart:convert';
|
||||
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
|
||||
String sha2(String input) {
|
||||
return sha256.convert(utf8.encode(input)).toString();
|
||||
}
|
||||
|
||||
Future<String> sha2Isolated(String input) async {
|
||||
if (input.length < 1048576) {
|
||||
// Less than 1MB
|
||||
return sha2(input);
|
||||
} else {
|
||||
return await compute(sha2, input);
|
||||
}
|
||||
}
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:langchain/langchain.dart';
|
||||
import 'package:weblibre/core/uuid.dart';
|
||||
import 'package:weblibre/features/geckoview/features/tabs/features/vector_store/utils/document_uuid.dart';
|
||||
|
||||
typedef DocumentParts = ({String mainDocumentId, List<Document> parts});
|
||||
|
||||
List<Document> _mergeShortDocuments(
|
||||
List<Document> documents,
|
||||
int maxLength, {
|
||||
// ignore: unused_element_parameter
|
||||
String separator = '\n',
|
||||
required Document Function(Document a, Document b) doMerge,
|
||||
int Function(String) lengthFunction = TextSplitter.defaultLengthFunction,
|
||||
}) {
|
||||
final result = <Document>[];
|
||||
|
||||
Document? current;
|
||||
for (final doc in documents) {
|
||||
if (current == null) {
|
||||
current = doc;
|
||||
} else if (lengthFunction(current.pageContent) +
|
||||
lengthFunction(doc.pageContent) <=
|
||||
maxLength) {
|
||||
current = doMerge(current, doc);
|
||||
} else {
|
||||
result.add(current);
|
||||
current = doc;
|
||||
}
|
||||
}
|
||||
|
||||
if (current != null) {
|
||||
result.add(current);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
List<String> _headerValueList(dynamic value) {
|
||||
return switch (value) {
|
||||
String _ => [value],
|
||||
List<String> _ => value,
|
||||
_ => throw Exception('Unsupported type'),
|
||||
};
|
||||
}
|
||||
|
||||
DocumentParts? splitMarkdownDocument(
|
||||
Document originalDoc, {
|
||||
required List<(String, String)> headersToSplitOn,
|
||||
required int chunkSize,
|
||||
required int chunkOverlap,
|
||||
}) {
|
||||
final markdownHeaderSplitter = MarkdownHeaderTextSplitter(
|
||||
stripHeaders: false,
|
||||
headersToSplitOn: headersToSplitOn,
|
||||
);
|
||||
|
||||
final markdownTextSplitter = MarkdownTextSplitter(
|
||||
chunkSize: chunkSize,
|
||||
chunkOverlap: chunkOverlap,
|
||||
);
|
||||
|
||||
final headerChunks = markdownHeaderSplitter.splitText(
|
||||
originalDoc.pageContent,
|
||||
);
|
||||
final docChunks = markdownTextSplitter.splitDocuments(headerChunks);
|
||||
|
||||
if (docChunks.length == 1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
final headerKeys = headersToSplitOn.map((header) => header.$2).toSet();
|
||||
final mergedDocChunks = _mergeShortDocuments(
|
||||
docChunks,
|
||||
chunkSize,
|
||||
doMerge: (a, b) {
|
||||
//Disallow merging into headers of the same
|
||||
final intersectingHeaders = a.metadata.keys
|
||||
.toSet()
|
||||
.intersection(b.metadata.keys.toSet())
|
||||
.intersection(headerKeys);
|
||||
|
||||
return Document(
|
||||
id: a.id ?? b.id,
|
||||
pageContent: '${a.pageContent}\n${b.pageContent}',
|
||||
metadata: {
|
||||
...a.metadata,
|
||||
...b.metadata,
|
||||
for (final header in intersectingHeaders)
|
||||
header:
|
||||
{
|
||||
..._headerValueList(a.metadata[header]),
|
||||
..._headerValueList(b.metadata[header]),
|
||||
}.toList(),
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
final mainDocumentId = originalDoc.id ?? uuid.v4();
|
||||
final documentIdFactory = DocumentUuid.fromUuid(mainDocumentId);
|
||||
|
||||
return (
|
||||
mainDocumentId: mainDocumentId,
|
||||
parts:
|
||||
mergedDocChunks
|
||||
.mapIndexed(
|
||||
(i, doc) => Document(
|
||||
id: documentIdFactory.getDocumentPartUuid(i),
|
||||
pageContent: doc.pageContent,
|
||||
metadata: mergeMaps(originalDoc.metadata, doc.metadata),
|
||||
),
|
||||
)
|
||||
.toList(),
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user