intermediate
This commit is contained in:
Fabian Freund
@@ -0,0 +1,6 @@
|
||||
abstract interface class IQueryBuilder {
|
||||
int get ftsTokenLimit;
|
||||
int get ftsMinTokenLength;
|
||||
|
||||
String buildFtsQuery(String input);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import 'package:exceptions/exceptions.dart';
|
||||
|
||||
abstract interface class ISearchSuggestionProvider {
|
||||
Future<Result<List<String>>> getSuggestions(String query);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
const barewordConcat = ' ';
|
||||
|
||||
sealed class Bareword {
|
||||
final String word;
|
||||
|
||||
@override
|
||||
int get hashCode => word.hashCode;
|
||||
|
||||
Bareword(String word) : word = word.replaceAll('"', '""');
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
if (other is Bareword) {
|
||||
return word == other.word;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
JoinedBareword join(Bareword other) {
|
||||
return JoinedBareword(other.word, this);
|
||||
}
|
||||
}
|
||||
|
||||
final class SimpleBareword extends Bareword {
|
||||
SimpleBareword(super.word);
|
||||
|
||||
@override
|
||||
String toString() => word;
|
||||
}
|
||||
|
||||
final class EnclosedBareword extends Bareword {
|
||||
EnclosedBareword(super.word);
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return '"$word"';
|
||||
}
|
||||
}
|
||||
|
||||
final class JoinedBareword extends Bareword {
|
||||
final Bareword parent;
|
||||
|
||||
JoinedBareword(super.bareword, this.parent);
|
||||
|
||||
bool hasDependency(Bareword other) {
|
||||
if (parent is JoinedBareword) {
|
||||
return (parent as JoinedBareword).hasDependency(other);
|
||||
} else {
|
||||
return parent == other;
|
||||
}
|
||||
}
|
||||
|
||||
Iterable<Bareword> get dependencies {
|
||||
final depencies = <Bareword>[parent];
|
||||
while (depencies.last is JoinedBareword) {
|
||||
depencies.add((depencies.last as JoinedBareword).parent);
|
||||
}
|
||||
|
||||
return depencies.reversed;
|
||||
}
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
final deps = dependencies
|
||||
.map((bareword) => bareword.toString())
|
||||
.join(barewordConcat);
|
||||
return '"$deps$barewordConcat$word"';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
import 'dart:math' as math;
|
||||
|
||||
import 'package:collection/collection.dart';
|
||||
import 'package:lensai/features/search/domain/entities/abstract/i_query_builder.dart';
|
||||
import 'package:lensai/features/search/domain/entities/bareword.dart';
|
||||
|
||||
typedef _Phrase = List<Bareword>;
|
||||
|
||||
sealed class FtsQueryBuilder {
|
||||
///Matching quoted strings like "xda bda" (group 1), as well as strings delimetered by
|
||||
///a whitespace (group 2)
|
||||
static final _tokenizePattern = RegExp('"([^"]+)"|([^ ]+)');
|
||||
|
||||
///Reserved FTS5 baewords
|
||||
static const _reservedBarewords = {'AND', 'OR', 'NOT'};
|
||||
|
||||
///Checks if string is an bareword accoring to:
|
||||
///https://www.sqlite.org/fts5.html#fts5_strings
|
||||
///
|
||||
///Elsewise ths string needs to get quoted
|
||||
static final _barewordPattern =
|
||||
RegExp(r'^([^\x00-\x7F]|[\w]|[\d]|[_]|[\x1A])+$');
|
||||
|
||||
static bool _isReservedBareword(Bareword input) =>
|
||||
_reservedBarewords.contains(input.word);
|
||||
|
||||
late final _Phrase _tokens;
|
||||
|
||||
bool get hasTokens => _tokens.isNotEmpty;
|
||||
|
||||
FtsQueryBuilder.tokenize({
|
||||
required String input,
|
||||
required int minTokenLength,
|
||||
required int tokenLimit,
|
||||
}) {
|
||||
final matches = _tokenizePattern.allMatches(input);
|
||||
|
||||
final barewords = matches
|
||||
.map((match) {
|
||||
if (match.group(1) != null) {
|
||||
return EnclosedBareword(match.group(1)!);
|
||||
} else if (_barewordPattern.hasMatch(match.group(2)!)) {
|
||||
return SimpleBareword(match.group(2)!);
|
||||
} else {
|
||||
return EnclosedBareword(match.group(2)!);
|
||||
}
|
||||
})
|
||||
.where((token) => token.word.isNotEmpty)
|
||||
.whereNot(_isReservedBareword)
|
||||
.toList();
|
||||
|
||||
//Merge short tokens
|
||||
_mergeShortBarewords(barewords, minTokenLength);
|
||||
|
||||
_tokens = barewords.take(tokenLimit).toList();
|
||||
}
|
||||
|
||||
static void _mergeShortBarewords(
|
||||
List<Bareword> barewords,
|
||||
int minTokenLength,
|
||||
) {
|
||||
for (var i = 0; i < barewords.length; i++) {
|
||||
final bareword = barewords[i];
|
||||
|
||||
if (i != 0) {
|
||||
if (bareword.word.length < minTokenLength) {
|
||||
barewords[i] = barewords[i - 1].join(bareword);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//Get weighted groups of tokens
|
||||
//Weight depends on order
|
||||
static List<_Phrase> _getWeighted(List<Bareword> tokens) {
|
||||
if (tokens.isEmpty) {
|
||||
return [];
|
||||
}
|
||||
|
||||
final phrases = List.generate(
|
||||
tokens.length - 1,
|
||||
(index) => tokens.sublist(0, tokens.length - index),
|
||||
);
|
||||
|
||||
phrases.addAll(tokens.map((token) => [token]));
|
||||
|
||||
return phrases;
|
||||
}
|
||||
|
||||
//Get all combinations of input tokens and treat them as equally
|
||||
//Match of bigger group will have greater score
|
||||
static List<_Phrase> _getCombinations(List<Bareword> tokens) {
|
||||
if (tokens.isEmpty) {
|
||||
return [];
|
||||
}
|
||||
|
||||
final combinations = <_Phrase>[];
|
||||
final slent = math.pow(2, tokens.length);
|
||||
|
||||
for (var i = 0; i < slent; i++) {
|
||||
final temp = <Bareword>[];
|
||||
|
||||
for (var j = 0; j < tokens.length; j++) {
|
||||
if ((i & math.pow(2, j).toInt()) > 0) {
|
||||
temp.add(tokens[j]);
|
||||
}
|
||||
}
|
||||
|
||||
if (temp.isNotEmpty) {
|
||||
combinations.add(temp);
|
||||
}
|
||||
}
|
||||
|
||||
return combinations;
|
||||
}
|
||||
|
||||
//If we have multiple unenclosed token, combine them into quoted to
|
||||
//get a better full range match
|
||||
//TODO: Before migration this was only used for non-prefix tokens, check for side effects
|
||||
static List<_Phrase> _combineUnenclosed(_Phrase phrase) {
|
||||
return [
|
||||
phrase,
|
||||
if (phrase.length > 2 &&
|
||||
phrase.every((bareword) => bareword is SimpleBareword))
|
||||
[EnclosedBareword(phrase.join(barewordConcat))],
|
||||
];
|
||||
}
|
||||
|
||||
//Decinding query strategy
|
||||
static List<_Phrase> _generatePhrases(List<Bareword> tokens) {
|
||||
if (tokens.length <= 4) {
|
||||
return _getCombinations(tokens);
|
||||
} else if (tokens.length <= 8) {
|
||||
return _getWeighted(tokens);
|
||||
} else {
|
||||
return [tokens];
|
||||
}
|
||||
}
|
||||
|
||||
static _Phrase _removeDependendBarewords(List<Bareword> phrase) {
|
||||
final joinedPhrase = List.of(phrase);
|
||||
|
||||
for (var i = 0; i < joinedPhrase.length; i++) {
|
||||
final bareword = joinedPhrase[i];
|
||||
|
||||
//Remove dependend phrases
|
||||
if (joinedPhrase.whereType<JoinedBareword>().any(
|
||||
(joined) => joined.dependencies.contains(bareword),
|
||||
)) {
|
||||
joinedPhrase.removeAt(i);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
|
||||
return joinedPhrase;
|
||||
}
|
||||
|
||||
String _buildQuery(
|
||||
List<Bareword> tokens,
|
||||
String Function(List<Bareword> phrase) barewordConcat,
|
||||
String phraseConcat,
|
||||
) {
|
||||
final phrases = _generatePhrases(tokens)
|
||||
.map(_removeDependendBarewords)
|
||||
.expand(_combineUnenclosed);
|
||||
|
||||
return phrases
|
||||
.where((phrase) => phrase.isNotEmpty)
|
||||
.sortedBy<num>((x) => x.length)
|
||||
.reversed
|
||||
.map(barewordConcat)
|
||||
.toSet()
|
||||
.join(phraseConcat);
|
||||
}
|
||||
|
||||
String build();
|
||||
}
|
||||
|
||||
final class PrefixQueryBuilder extends FtsQueryBuilder {
|
||||
PrefixQueryBuilder.tokenize({
|
||||
required super.input,
|
||||
required super.minTokenLength,
|
||||
required super.tokenLimit,
|
||||
}) : super.tokenize();
|
||||
|
||||
@override
|
||||
String build() {
|
||||
return _buildQuery(
|
||||
_tokens,
|
||||
(phrase) {
|
||||
if (phrase.length == 1) {
|
||||
return '${phrase.first}*';
|
||||
}
|
||||
|
||||
return 'NEAR(${phrase.map((bareword) => '$bareword*').join(' ')})';
|
||||
},
|
||||
' OR ',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
final class TrigramQueryBuilder extends FtsQueryBuilder {
|
||||
TrigramQueryBuilder.tokenize({
|
||||
required super.input,
|
||||
required super.minTokenLength,
|
||||
required super.tokenLimit,
|
||||
}) : super.tokenize();
|
||||
|
||||
@override
|
||||
String build() {
|
||||
return _buildQuery(
|
||||
_tokens,
|
||||
(phrase) => phrase.join(' '),
|
||||
' OR ',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
mixin PrefixQueryBuilderMixin implements IQueryBuilder {
|
||||
@override
|
||||
String buildFtsQuery(String input) {
|
||||
final queryBuilder = PrefixQueryBuilder.tokenize(
|
||||
input: input,
|
||||
minTokenLength: ftsMinTokenLength,
|
||||
tokenLimit: ftsTokenLimit,
|
||||
);
|
||||
|
||||
return queryBuilder.build();
|
||||
}
|
||||
}
|
||||
|
||||
mixin TrigramQueryBuilderMixin implements IQueryBuilder {
|
||||
@override
|
||||
String buildFtsQuery(String input) {
|
||||
final queryBuilder = TrigramQueryBuilder.tokenize(
|
||||
input: input,
|
||||
minTokenLength: ftsMinTokenLength,
|
||||
tokenLimit: ftsTokenLimit,
|
||||
);
|
||||
|
||||
return queryBuilder.build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import 'package:lensai/features/kagi/data/services/autosuggest.dart';
|
||||
import 'package:lensai/features/kagi/domain/repositories/search_suggestions.dart';
|
||||
import 'package:lensai/features/search/domain/entities/abstract/i_search_suggestion_provider.dart';
|
||||
import 'package:riverpod/riverpod.dart';
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
|
||||
part 'search_suggestions.g.dart';
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
ISearchSuggestionProvider defaultSearchSuggestions(Ref ref) {
|
||||
return ref.watch(kagiAutosuggestServiceProvider.notifier);
|
||||
}
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
class SearchSuggestions extends _$SearchSuggestions {
|
||||
late void Function(String query) _addQueryBinding;
|
||||
|
||||
void addQuery(String query) => _addQueryBinding(query);
|
||||
|
||||
@override
|
||||
Raw<Stream<List<String>>> build({
|
||||
ISearchSuggestionProvider? suggestionsProvider,
|
||||
}) {
|
||||
final defaultProvider = ref.watch(defaultSearchSuggestionsProvider);
|
||||
final resolvedProvider = suggestionsProvider ?? defaultProvider;
|
||||
|
||||
_addQueryBinding = ref
|
||||
.watch(searchSuggestionsRepositoryProvider(resolvedProvider).notifier)
|
||||
.addQuery;
|
||||
|
||||
return ref.watch(
|
||||
searchSuggestionsRepositoryProvider(resolvedProvider),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'search_suggestions.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
String _$defaultSearchSuggestionsHash() =>
|
||||
r'56514064baa64acf0a28ec34cfad957dd863538b';
|
||||
|
||||
/// See also [defaultSearchSuggestions].
|
||||
@ProviderFor(defaultSearchSuggestions)
|
||||
final defaultSearchSuggestionsProvider =
|
||||
Provider<ISearchSuggestionProvider>.internal(
|
||||
defaultSearchSuggestions,
|
||||
name: r'defaultSearchSuggestionsProvider',
|
||||
debugGetCreateSourceHash: const bool.fromEnvironment('dart.vm.product')
|
||||
? null
|
||||
: _$defaultSearchSuggestionsHash,
|
||||
dependencies: null,
|
||||
allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@Deprecated('Will be removed in 3.0. Use Ref instead')
|
||||
// ignore: unused_element
|
||||
typedef DefaultSearchSuggestionsRef = ProviderRef<ISearchSuggestionProvider>;
|
||||
String _$searchSuggestionsHash() => r'ecc378aca325cc6759292a06d8d4d5ace910eb7f';
|
||||
|
||||
/// Copied from Dart SDK
|
||||
class _SystemHash {
|
||||
_SystemHash._();
|
||||
|
||||
static int combine(int hash, int value) {
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + value);
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + ((0x0007ffff & hash) << 10));
|
||||
return hash ^ (hash >> 6);
|
||||
}
|
||||
|
||||
static int finish(int hash) {
|
||||
// ignore: parameter_assignments
|
||||
hash = 0x1fffffff & (hash + ((0x03ffffff & hash) << 3));
|
||||
// ignore: parameter_assignments
|
||||
hash = hash ^ (hash >> 11);
|
||||
return 0x1fffffff & (hash + ((0x00003fff & hash) << 15));
|
||||
}
|
||||
}
|
||||
|
||||
abstract class _$SearchSuggestions
|
||||
extends BuildlessNotifier<Raw<Stream<List<String>>>> {
|
||||
late final ISearchSuggestionProvider? suggestionsProvider;
|
||||
|
||||
Raw<Stream<List<String>>> build({
|
||||
ISearchSuggestionProvider? suggestionsProvider,
|
||||
});
|
||||
}
|
||||
|
||||
/// See also [SearchSuggestions].
|
||||
@ProviderFor(SearchSuggestions)
|
||||
const searchSuggestionsProvider = SearchSuggestionsFamily();
|
||||
|
||||
/// See also [SearchSuggestions].
|
||||
class SearchSuggestionsFamily extends Family<Raw<Stream<List<String>>>> {
|
||||
/// See also [SearchSuggestions].
|
||||
const SearchSuggestionsFamily();
|
||||
|
||||
/// See also [SearchSuggestions].
|
||||
SearchSuggestionsProvider call({
|
||||
ISearchSuggestionProvider? suggestionsProvider,
|
||||
}) {
|
||||
return SearchSuggestionsProvider(
|
||||
suggestionsProvider: suggestionsProvider,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
SearchSuggestionsProvider getProviderOverride(
|
||||
covariant SearchSuggestionsProvider provider,
|
||||
) {
|
||||
return call(
|
||||
suggestionsProvider: provider.suggestionsProvider,
|
||||
);
|
||||
}
|
||||
|
||||
static const Iterable<ProviderOrFamily>? _dependencies = null;
|
||||
|
||||
@override
|
||||
Iterable<ProviderOrFamily>? get dependencies => _dependencies;
|
||||
|
||||
static const Iterable<ProviderOrFamily>? _allTransitiveDependencies = null;
|
||||
|
||||
@override
|
||||
Iterable<ProviderOrFamily>? get allTransitiveDependencies =>
|
||||
_allTransitiveDependencies;
|
||||
|
||||
@override
|
||||
String? get name => r'searchSuggestionsProvider';
|
||||
}
|
||||
|
||||
/// See also [SearchSuggestions].
|
||||
class SearchSuggestionsProvider
|
||||
extends NotifierProviderImpl<SearchSuggestions, Raw<Stream<List<String>>>> {
|
||||
/// See also [SearchSuggestions].
|
||||
SearchSuggestionsProvider({
|
||||
ISearchSuggestionProvider? suggestionsProvider,
|
||||
}) : this._internal(
|
||||
() => SearchSuggestions()..suggestionsProvider = suggestionsProvider,
|
||||
from: searchSuggestionsProvider,
|
||||
name: r'searchSuggestionsProvider',
|
||||
debugGetCreateSourceHash:
|
||||
const bool.fromEnvironment('dart.vm.product')
|
||||
? null
|
||||
: _$searchSuggestionsHash,
|
||||
dependencies: SearchSuggestionsFamily._dependencies,
|
||||
allTransitiveDependencies:
|
||||
SearchSuggestionsFamily._allTransitiveDependencies,
|
||||
suggestionsProvider: suggestionsProvider,
|
||||
);
|
||||
|
||||
SearchSuggestionsProvider._internal(
|
||||
super._createNotifier, {
|
||||
required super.name,
|
||||
required super.dependencies,
|
||||
required super.allTransitiveDependencies,
|
||||
required super.debugGetCreateSourceHash,
|
||||
required super.from,
|
||||
required this.suggestionsProvider,
|
||||
}) : super.internal();
|
||||
|
||||
final ISearchSuggestionProvider? suggestionsProvider;
|
||||
|
||||
@override
|
||||
Raw<Stream<List<String>>> runNotifierBuild(
|
||||
covariant SearchSuggestions notifier,
|
||||
) {
|
||||
return notifier.build(
|
||||
suggestionsProvider: suggestionsProvider,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Override overrideWith(SearchSuggestions Function() create) {
|
||||
return ProviderOverride(
|
||||
origin: this,
|
||||
override: SearchSuggestionsProvider._internal(
|
||||
() => create()..suggestionsProvider = suggestionsProvider,
|
||||
from: from,
|
||||
name: null,
|
||||
dependencies: null,
|
||||
allTransitiveDependencies: null,
|
||||
debugGetCreateSourceHash: null,
|
||||
suggestionsProvider: suggestionsProvider,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
NotifierProviderElement<SearchSuggestions, Raw<Stream<List<String>>>>
|
||||
createElement() {
|
||||
return _SearchSuggestionsProviderElement(this);
|
||||
}
|
||||
|
||||
@override
|
||||
bool operator ==(Object other) {
|
||||
return other is SearchSuggestionsProvider &&
|
||||
other.suggestionsProvider == suggestionsProvider;
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode {
|
||||
var hash = _SystemHash.combine(0, runtimeType.hashCode);
|
||||
hash = _SystemHash.combine(hash, suggestionsProvider.hashCode);
|
||||
|
||||
return _SystemHash.finish(hash);
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated('Will be removed in 3.0. Use Ref instead')
|
||||
// ignore: unused_element
|
||||
mixin SearchSuggestionsRef on NotifierProviderRef<Raw<Stream<List<String>>>> {
|
||||
/// The parameter `suggestionsProvider` of this provider.
|
||||
ISearchSuggestionProvider? get suggestionsProvider;
|
||||
}
|
||||
|
||||
class _SearchSuggestionsProviderElement extends NotifierProviderElement<
|
||||
SearchSuggestions, Raw<Stream<List<String>>>> with SearchSuggestionsRef {
|
||||
_SearchSuggestionsProviderElement(super.provider);
|
||||
|
||||
@override
|
||||
ISearchSuggestionProvider? get suggestionsProvider =>
|
||||
(origin as SearchSuggestionsProvider).suggestionsProvider;
|
||||
}
|
||||
// ignore_for_file: type=lint
|
||||
// ignore_for_file: subtype_of_sealed_class, invalid_use_of_internal_member, invalid_use_of_visible_for_testing_member, deprecated_member_use_from_same_package
|
||||
@@ -0,0 +1,68 @@
|
||||
import 'package:lensai/features/search/domain/entities/bareword.dart';
|
||||
|
||||
typedef _Phrase = List<Bareword>;
|
||||
|
||||
sealed class UnixTokenizer {
|
||||
///Matching quoted strings like "xda bda" (group 1), as well as strings delimetered by
|
||||
///a whitespace (group 2)
|
||||
static final _tokenizePattern = RegExp('"([^"]+)"|([^ ]+)');
|
||||
|
||||
late final _Phrase _tokens;
|
||||
|
||||
bool get hasTokens => _tokens.isNotEmpty;
|
||||
|
||||
static void _mergeShortBarewords(
|
||||
List<Bareword> barewords,
|
||||
int minTokenLength,
|
||||
) {
|
||||
for (var i = 0; i < barewords.length; i++) {
|
||||
final bareword = barewords[i];
|
||||
|
||||
if (i != 0) {
|
||||
if (bareword.word.length < minTokenLength) {
|
||||
barewords[i] = barewords[i - 1].join(bareword);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
UnixTokenizer.tokenize({
|
||||
required String input,
|
||||
required int minTokenLength,
|
||||
required int tokenLimit,
|
||||
}) {
|
||||
final matches = _tokenizePattern.allMatches(input);
|
||||
|
||||
final barewords = matches
|
||||
.map((match) {
|
||||
if (match.group(1) != null) {
|
||||
return EnclosedBareword(match.group(1)!);
|
||||
} else {
|
||||
return SimpleBareword(match.group(2)!);
|
||||
}
|
||||
})
|
||||
.where((token) => token.word.isNotEmpty)
|
||||
.toList();
|
||||
|
||||
//Merge short tokens
|
||||
_mergeShortBarewords(barewords, minTokenLength);
|
||||
|
||||
_tokens = barewords.take(tokenLimit).toList();
|
||||
}
|
||||
|
||||
String build({bool wildcard});
|
||||
}
|
||||
|
||||
final class UnixLikeQueryBuilder extends UnixTokenizer {
|
||||
UnixLikeQueryBuilder.tokenize({
|
||||
required super.input,
|
||||
required super.minTokenLength,
|
||||
required super.tokenLimit,
|
||||
}) : super.tokenize();
|
||||
|
||||
@override
|
||||
String build({bool wildcard = true}) {
|
||||
final joined = _tokens.join('%');
|
||||
return wildcard ? '%$joined%' : joined;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user