prepare for multiple apps

This commit is contained in:
Fabian Freund
2026-04-06 12:23:11 +02:00
parent bd1600e8dc
commit 5afc323f04
904 changed files with 29 additions and 29 deletions
@@ -0,0 +1,61 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'dart:convert';
import 'package:exceptions/exceptions.dart';
import 'package:http/http.dart' as http;
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:weblibre/core/http_error_handler.dart';
import 'package:weblibre/features/search/domain/entities/abstract/i_search_suggestion_provider.dart';
part 'brave.g.dart';
@Riverpod(keepAlive: true)
class BraveAutosuggestService extends _$BraveAutosuggestService
implements ISearchSuggestionProvider {
static final _baseUrl = Uri.https('search.brave.com', 'api/suggest');
late http.Client _client;
@override
void build() {
_client = http.Client();
ref.onDispose(() {
_client.close();
});
}
@override
Future<Result<List<String>>> getSuggestions(String query) {
return Result.fromAsync(() async {
final response = await _client
.get(_baseUrl.replace(queryParameters: {'q': query}))
.timeout(const Duration(seconds: 5));
final results = jsonDecode(utf8.decode(response.bodyBytes)) as List;
return switch (results.last) {
final String result => [result],
final List resultList => resultList.cast(),
_ => [],
};
}, exceptionHandler: handleHttpError);
}
}
@@ -0,0 +1,63 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'brave.dart';
// **************************************************************************
// RiverpodGenerator
// **************************************************************************
// GENERATED CODE - DO NOT MODIFY BY HAND
// ignore_for_file: type=lint, type=warning
@ProviderFor(BraveAutosuggestService)
final braveAutosuggestServiceProvider = BraveAutosuggestServiceProvider._();
final class BraveAutosuggestServiceProvider
extends $NotifierProvider<BraveAutosuggestService, void> {
BraveAutosuggestServiceProvider._()
: super(
from: null,
argument: null,
retry: null,
name: r'braveAutosuggestServiceProvider',
isAutoDispose: false,
dependencies: null,
$allTransitiveDependencies: null,
);
@override
String debugGetCreateSourceHash() => _$braveAutosuggestServiceHash();
@$internal
@override
BraveAutosuggestService create() => BraveAutosuggestService();
/// {@macro riverpod.override_with_value}
Override overrideWithValue(void value) {
return $ProviderOverride(
origin: this,
providerOverride: $SyncValueProvider<void>(value),
);
}
}
String _$braveAutosuggestServiceHash() =>
r'3b2f465dcf4b44b0450975d3909fc7066d171bf2';
abstract class _$BraveAutosuggestService extends $Notifier<void> {
void build();
@$mustCallSuper
@override
void runBuild() {
final ref = this.ref as $Ref<void, void>;
final element =
ref.element
as $ClassProviderElement<
AnyNotifier<void, void>,
void,
Object?,
Object?
>;
element.handleCreate(ref, build);
}
}
@@ -0,0 +1,61 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'dart:convert';
import 'package:exceptions/exceptions.dart';
import 'package:http/http.dart' as http;
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:weblibre/core/http_error_handler.dart';
import 'package:weblibre/features/search/domain/entities/abstract/i_search_suggestion_provider.dart';
part 'duckduckgo.g.dart';
@Riverpod(keepAlive: true)
class DuckDuckGoAutosuggestService extends _$DuckDuckGoAutosuggestService
implements ISearchSuggestionProvider {
static final _baseUrl = Uri.https('duckduckgo.com', 'ac/');
late http.Client _client;
@override
void build() {
_client = http.Client();
ref.onDispose(() {
_client.close();
});
}
@override
Future<Result<List<String>>> getSuggestions(String query) {
return Result.fromAsync(() async {
final response = await _client
.get(_baseUrl.replace(queryParameters: {'type': 'list', 'q': query}))
.timeout(const Duration(seconds: 5));
final results = jsonDecode(utf8.decode(response.bodyBytes)) as List;
return switch (results.last) {
final String result => [result],
final List resultList => resultList.cast(),
_ => [],
};
}, exceptionHandler: handleHttpError);
}
}
@@ -0,0 +1,64 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'duckduckgo.dart';
// **************************************************************************
// RiverpodGenerator
// **************************************************************************
// GENERATED CODE - DO NOT MODIFY BY HAND
// ignore_for_file: type=lint, type=warning
@ProviderFor(DuckDuckGoAutosuggestService)
final duckDuckGoAutosuggestServiceProvider =
DuckDuckGoAutosuggestServiceProvider._();
final class DuckDuckGoAutosuggestServiceProvider
extends $NotifierProvider<DuckDuckGoAutosuggestService, void> {
DuckDuckGoAutosuggestServiceProvider._()
: super(
from: null,
argument: null,
retry: null,
name: r'duckDuckGoAutosuggestServiceProvider',
isAutoDispose: false,
dependencies: null,
$allTransitiveDependencies: null,
);
@override
String debugGetCreateSourceHash() => _$duckDuckGoAutosuggestServiceHash();
@$internal
@override
DuckDuckGoAutosuggestService create() => DuckDuckGoAutosuggestService();
/// {@macro riverpod.override_with_value}
Override overrideWithValue(void value) {
return $ProviderOverride(
origin: this,
providerOverride: $SyncValueProvider<void>(value),
);
}
}
String _$duckDuckGoAutosuggestServiceHash() =>
r'd9f30a577b276049b252be890bc73eb18aa118f2';
abstract class _$DuckDuckGoAutosuggestService extends $Notifier<void> {
void build();
@$mustCallSuper
@override
void runBuild() {
final ref = this.ref as $Ref<void, void>;
final element =
ref.element
as $ClassProviderElement<
AnyNotifier<void, void>,
void,
Object?,
Object?
>;
element.handleCreate(ref, build);
}
}
@@ -0,0 +1,36 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'package:exceptions/exceptions.dart';
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:weblibre/features/search/domain/entities/abstract/i_search_suggestion_provider.dart';
part 'empty.g.dart';
@Riverpod(keepAlive: true)
class EmptyAutosuggestService extends _$EmptyAutosuggestService
implements ISearchSuggestionProvider {
@override
void build() {}
@override
Future<Result<List<String>>> getSuggestions(String query) {
return Future.value(Result.success([]));
}
}
@@ -0,0 +1,63 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'empty.dart';
// **************************************************************************
// RiverpodGenerator
// **************************************************************************
// GENERATED CODE - DO NOT MODIFY BY HAND
// ignore_for_file: type=lint, type=warning
@ProviderFor(EmptyAutosuggestService)
final emptyAutosuggestServiceProvider = EmptyAutosuggestServiceProvider._();
final class EmptyAutosuggestServiceProvider
extends $NotifierProvider<EmptyAutosuggestService, void> {
EmptyAutosuggestServiceProvider._()
: super(
from: null,
argument: null,
retry: null,
name: r'emptyAutosuggestServiceProvider',
isAutoDispose: false,
dependencies: null,
$allTransitiveDependencies: null,
);
@override
String debugGetCreateSourceHash() => _$emptyAutosuggestServiceHash();
@$internal
@override
EmptyAutosuggestService create() => EmptyAutosuggestService();
/// {@macro riverpod.override_with_value}
Override overrideWithValue(void value) {
return $ProviderOverride(
origin: this,
providerOverride: $SyncValueProvider<void>(value),
);
}
}
String _$emptyAutosuggestServiceHash() =>
r'7fbe81807baf379cf621ef46fba00e46778978f8';
abstract class _$EmptyAutosuggestService extends $Notifier<void> {
void build();
@$mustCallSuper
@override
void runBuild() {
final ref = this.ref as $Ref<void, void>;
final element =
ref.element
as $ClassProviderElement<
AnyNotifier<void, void>,
void,
Object?,
Object?
>;
element.handleCreate(ref, build);
}
}
@@ -0,0 +1,61 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'dart:convert';
import 'package:exceptions/exceptions.dart';
import 'package:http/http.dart' as http;
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:weblibre/core/http_error_handler.dart';
import 'package:weblibre/features/search/domain/entities/abstract/i_search_suggestion_provider.dart';
part 'kagi.g.dart';
@Riverpod(keepAlive: true)
class KagiAutosuggestService extends _$KagiAutosuggestService
implements ISearchSuggestionProvider {
static final _baseUrl = Uri.https('kagi.com', 'api/autosuggest');
late http.Client _client;
@override
void build() {
_client = http.Client();
ref.onDispose(() {
_client.close();
});
}
@override
Future<Result<List<String>>> getSuggestions(String query) {
return Result.fromAsync(() async {
final response = await _client
.get(_baseUrl.replace(queryParameters: {'q': query}))
.timeout(const Duration(seconds: 5));
final results = jsonDecode(utf8.decode(response.bodyBytes)) as List;
return switch (results.last) {
final String result => [result],
final List resultList => resultList.cast(),
_ => [],
};
}, exceptionHandler: handleHttpError);
}
}
@@ -0,0 +1,63 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'kagi.dart';
// **************************************************************************
// RiverpodGenerator
// **************************************************************************
// GENERATED CODE - DO NOT MODIFY BY HAND
// ignore_for_file: type=lint, type=warning
@ProviderFor(KagiAutosuggestService)
final kagiAutosuggestServiceProvider = KagiAutosuggestServiceProvider._();
final class KagiAutosuggestServiceProvider
extends $NotifierProvider<KagiAutosuggestService, void> {
KagiAutosuggestServiceProvider._()
: super(
from: null,
argument: null,
retry: null,
name: r'kagiAutosuggestServiceProvider',
isAutoDispose: false,
dependencies: null,
$allTransitiveDependencies: null,
);
@override
String debugGetCreateSourceHash() => _$kagiAutosuggestServiceHash();
@$internal
@override
KagiAutosuggestService create() => KagiAutosuggestService();
/// {@macro riverpod.override_with_value}
Override overrideWithValue(void value) {
return $ProviderOverride(
origin: this,
providerOverride: $SyncValueProvider<void>(value),
);
}
}
String _$kagiAutosuggestServiceHash() =>
r'039fe84dd480316eab38c80b41e16d16d6c14a7c';
abstract class _$KagiAutosuggestService extends $Notifier<void> {
void build();
@$mustCallSuper
@override
void runBuild() {
final ref = this.ref as $Ref<void, void>;
final element =
ref.element
as $ClassProviderElement<
AnyNotifier<void, void>,
void,
Object?,
Object?
>;
element.handleCreate(ref, build);
}
}
@@ -0,0 +1,66 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// ignore_for_file: avoid_dynamic_calls
import 'dart:convert';
import 'package:exceptions/exceptions.dart';
import 'package:http/http.dart' as http;
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:weblibre/core/http_error_handler.dart';
import 'package:weblibre/features/search/domain/entities/abstract/i_search_suggestion_provider.dart';
part 'qwant.g.dart';
@Riverpod(keepAlive: true)
class QwantAutosuggestService extends _$QwantAutosuggestService
implements ISearchSuggestionProvider {
static final _baseUrl = Uri.https('api.qwant.com', 'v3/suggest');
late http.Client _client;
@override
void build() {
_client = http.Client();
ref.onDispose(() {
_client.close();
});
}
@override
Future<Result<List<String>>> getSuggestions(String query) {
return Result.fromAsync(() async {
final response = await _client
.get(_baseUrl.replace(queryParameters: {'q': query}))
.timeout(const Duration(seconds: 5));
final results =
jsonDecode(utf8.decode(response.bodyBytes)) as Map<String, dynamic>;
if (results['status'] != 'success') {
throw Exception('qwant api error');
}
final items = results['data']['items'] as List;
return items.map((item) => item['value'] as String).toList();
}, exceptionHandler: handleHttpError);
}
}
@@ -0,0 +1,63 @@
// GENERATED CODE - DO NOT MODIFY BY HAND
part of 'qwant.dart';
// **************************************************************************
// RiverpodGenerator
// **************************************************************************
// GENERATED CODE - DO NOT MODIFY BY HAND
// ignore_for_file: type=lint, type=warning
@ProviderFor(QwantAutosuggestService)
final qwantAutosuggestServiceProvider = QwantAutosuggestServiceProvider._();
final class QwantAutosuggestServiceProvider
extends $NotifierProvider<QwantAutosuggestService, void> {
QwantAutosuggestServiceProvider._()
: super(
from: null,
argument: null,
retry: null,
name: r'qwantAutosuggestServiceProvider',
isAutoDispose: false,
dependencies: null,
$allTransitiveDependencies: null,
);
@override
String debugGetCreateSourceHash() => _$qwantAutosuggestServiceHash();
@$internal
@override
QwantAutosuggestService create() => QwantAutosuggestService();
/// {@macro riverpod.override_with_value}
Override overrideWithValue(void value) {
return $ProviderOverride(
origin: this,
providerOverride: $SyncValueProvider<void>(value),
);
}
}
String _$qwantAutosuggestServiceHash() =>
r'a942cd073289d20d43647de5c0f43fb08790045f';
abstract class _$QwantAutosuggestService extends $Notifier<void> {
void build();
@$mustCallSuper
@override
void runBuild() {
final ref = this.ref as $Ref<void, void>;
final element =
ref.element
as $ClassProviderElement<
AnyNotifier<void, void>,
void,
Object?,
Object?
>;
element.handleCreate(ref, build);
}
}
@@ -0,0 +1,26 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
abstract interface class IQueryBuilder {
int get ftsTokenLimit;
int get ftsMinTokenLength;
String buildFtsQuery(String input);
String buildLikeQuery(String input);
}
@@ -0,0 +1,39 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'package:exceptions/exceptions.dart';
import 'package:weblibre/features/bangs/data/models/bang_group.dart';
import 'package:weblibre/features/bangs/data/models/bang_key.dart';
enum SearchSuggestionProviders {
none('Disabled', null),
brave('Brave', BangKey(group: BangGroup.general, trigger: 'brave')),
ddg('DuckDuckGo', BangKey(group: BangGroup.general, trigger: 'ddg')),
kagi('Kagi', BangKey(group: BangGroup.kagi, trigger: 'kagi')),
qwant('Qwant', BangKey(group: BangGroup.general, trigger: 'qwant'));
final String label;
final BangKey? relatedBang;
const SearchSuggestionProviders(this.label, this.relatedBang);
}
abstract interface class ISearchSuggestionProvider {
Future<Result<List<String>>> getSuggestions(String query);
}
@@ -0,0 +1,89 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
const barewordConcat = ' ';
sealed class Bareword {
final String word;
@override
int get hashCode => word.hashCode;
Bareword(String word) : word = word.replaceAll('"', '""');
@override
bool operator ==(Object other) {
if (other is Bareword) {
return word == other.word;
}
return false;
}
JoinedBareword join(Bareword other) {
return JoinedBareword(other.word, this);
}
}
final class SimpleBareword extends Bareword {
SimpleBareword(super.word);
@override
String toString() => word;
}
final class EnclosedBareword extends Bareword {
EnclosedBareword(super.word);
@override
String toString() {
return '"$word"';
}
}
final class JoinedBareword extends Bareword {
final Bareword parent;
JoinedBareword(super.bareword, this.parent);
bool hasDependency(Bareword other) {
if (parent is JoinedBareword) {
return (parent as JoinedBareword).hasDependency(other);
} else {
return parent == other;
}
}
Iterable<Bareword> get dependencies {
final depencies = <Bareword>[parent];
while (depencies.last is JoinedBareword) {
depencies.add((depencies.last as JoinedBareword).parent);
}
return depencies.reversed;
}
@override
String toString() {
final deps = dependencies
.map((bareword) => bareword.toString())
.join(barewordConcat);
return '"$deps$barewordConcat$word"';
}
}
@@ -0,0 +1,297 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'dart:math' as math;
import 'package:collection/collection.dart';
import 'package:weblibre/features/search/domain/entities/abstract/i_query_builder.dart';
import 'package:weblibre/features/search/domain/entities/bareword.dart';
import 'package:weblibre/features/search/domain/unix_tokenizer.dart';
typedef _Phrase = List<Bareword>;
sealed class FtsQueryBuilder {
///Matching quoted strings like "xda bda" (group 1), as well as strings delimetered by
///a whitespace (group 2)
static final _tokenizePattern = RegExp('"([^"]+)"|([^ ]+)');
///Reserved FTS5 baewords
static const _reservedBarewords = {'AND', 'OR', 'NOT'};
///Checks if string is an bareword accoring to:
///https://www.sqlite.org/fts5.html#fts5_strings
///
///Elsewise ths string needs to get quoted
static final _barewordPattern = RegExp(
r'^([^\x00-\x7F]|[\w]|[\d]|[_]|[\x1A])+$',
);
static bool _isReservedBareword(Bareword input) =>
_reservedBarewords.contains(input.word);
late final _Phrase _tokens;
bool get hasTokens => _tokens.isNotEmpty;
FtsQueryBuilder.tokenize({
required String input,
required int minTokenLength,
required int tokenLimit,
}) {
final matches = _tokenizePattern.allMatches(input);
final barewords = matches
.map((match) {
if (match.group(1) != null) {
return EnclosedBareword(match.group(1)!);
} else if (_barewordPattern.hasMatch(match.group(2)!)) {
return SimpleBareword(match.group(2)!);
} else {
return EnclosedBareword(match.group(2)!);
}
})
.where((token) => token.word.isNotEmpty)
.whereNot(_isReservedBareword)
.toList();
//Merge short tokens
_mergeShortBarewords(barewords, minTokenLength);
_tokens = barewords
.where((bareword) => bareword.word.length >= minTokenLength)
.take(tokenLimit)
.toList();
}
static void _mergeShortBarewords(
List<Bareword> barewords,
int minTokenLength,
) {
for (var i = 0; i < barewords.length; i++) {
final bareword = barewords[i];
if (i > 0) {
if (bareword.word.length < minTokenLength) {
barewords[i] = barewords[i - 1].join(bareword);
}
}
}
}
//Get weighted groups of tokens
//Weight depends on order
static List<_Phrase> _getWeighted(List<Bareword> tokens) {
if (tokens.isEmpty) {
return [];
}
final phrases = List.generate(
tokens.length - 1,
(index) => tokens.sublist(0, tokens.length - index),
);
phrases.addAll(tokens.map((token) => [token]));
return phrases;
}
//Get all combinations of input tokens and treat them as equally
//Match of bigger group will have greater score
static List<_Phrase> _getCombinations(List<Bareword> tokens) {
if (tokens.isEmpty) {
return [];
}
final combinations = <_Phrase>[];
final slent = math.pow(2, tokens.length);
for (var i = 0; i < slent; i++) {
final temp = <Bareword>[];
for (var j = 0; j < tokens.length; j++) {
if ((i & math.pow(2, j).toInt()) > 0) {
temp.add(tokens[j]);
}
}
if (temp.isNotEmpty) {
combinations.add(temp);
}
}
return combinations;
}
//If we have multiple unenclosed token, combine them into quoted to
//get a better full range match
//TODO: Before migration this was only used for non-prefix tokens, check for side effects
static List<_Phrase> _combineUnenclosed(_Phrase phrase) {
return [
phrase,
if (phrase.length > 2 &&
phrase.every((bareword) => bareword is SimpleBareword))
[EnclosedBareword(phrase.join(barewordConcat))],
];
}
//Decinding query strategy
static List<_Phrase> _generatePhrases(List<Bareword> tokens) {
if (tokens.length <= 4) {
return _getCombinations(tokens);
} else if (tokens.length <= 8) {
return _getWeighted(tokens);
} else {
return [tokens];
}
}
static _Phrase _removeDependendBarewords(List<Bareword> phrase) {
final joinedPhrase = List.of(phrase);
for (var i = 0; i < joinedPhrase.length; i++) {
final bareword = joinedPhrase[i];
//Remove dependend phrases
if (joinedPhrase.whereType<JoinedBareword>().any(
(joined) => joined.dependencies.contains(bareword),
)) {
joinedPhrase.removeAt(i);
i--;
}
}
return joinedPhrase;
}
String _buildQuery(
List<Bareword> tokens,
String Function(List<Bareword> phrase) barewordConcat,
String phraseConcat,
) {
final phrases = _generatePhrases(
tokens,
).map(_removeDependendBarewords).expand(_combineUnenclosed);
return phrases
.where((phrase) => phrase.isNotEmpty)
.sortedBy<num>((x) => x.length)
.reversed
.map(barewordConcat)
.toSet()
.join(phraseConcat);
}
String build();
}
final class PrefixQueryBuilder extends FtsQueryBuilder {
PrefixQueryBuilder.tokenize({
required super.input,
required super.minTokenLength,
required super.tokenLimit,
}) : super.tokenize();
@override
String build() {
return _buildQuery(_tokens, (phrase) {
if (phrase.length == 1) {
return '${phrase.first}*';
}
return 'NEAR(${phrase.map((bareword) => '$bareword*').join(' ')})';
}, ' OR ');
}
}
final class TrigramQueryBuilder extends FtsQueryBuilder {
TrigramQueryBuilder.tokenize({
required super.input,
required super.minTokenLength,
required super.tokenLimit,
}) : super.tokenize();
@override
String build() {
return _buildQuery(_tokens, (phrase) => phrase.join(' '), ' OR ');
}
}
mixin PrefixQueryBuilderMixin implements IQueryBuilder {
@override
String buildFtsQuery(String input) {
final queryBuilder = PrefixQueryBuilder.tokenize(
input: input,
minTokenLength: ftsMinTokenLength,
tokenLimit: ftsTokenLimit,
);
if (queryBuilder.hasTokens) {
return queryBuilder.build();
}
return '';
}
@override
String buildLikeQuery(String input) {
final likeQueryBuilder = UnixLikeQueryBuilder.tokenize(
input: input,
minTokenLength: 1,
tokenLimit: 5,
);
if (likeQueryBuilder.hasTokens) {
return likeQueryBuilder.build();
}
return '';
}
}
mixin TrigramQueryBuilderMixin implements IQueryBuilder {
@override
String buildFtsQuery(String input) {
final queryBuilder = TrigramQueryBuilder.tokenize(
input: input,
minTokenLength: ftsMinTokenLength,
tokenLimit: ftsTokenLimit,
);
if (queryBuilder.hasTokens) {
return queryBuilder.build();
}
return '';
}
@override
String buildLikeQuery(String input) {
final likeQueryBuilder = UnixLikeQueryBuilder.tokenize(
input: input,
minTokenLength: 1,
tokenLimit: 5,
);
if (likeQueryBuilder.hasTokens) {
return likeQueryBuilder.build();
}
return '';
}
}
@@ -0,0 +1,87 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
import 'package:weblibre/features/search/domain/entities/bareword.dart';
typedef _Phrase = List<Bareword>;
sealed class UnixTokenizer {
///Matching quoted strings like "xda bda" (group 1), as well as strings delimetered by
///a whitespace (group 2)
static final _tokenizePattern = RegExp('"([^"]+)"|([^ ]+)');
late final _Phrase _tokens;
bool get hasTokens => _tokens.isNotEmpty;
static void _mergeShortBarewords(
List<Bareword> barewords,
int minTokenLength,
) {
for (var i = 0; i < barewords.length; i++) {
final bareword = barewords[i];
if (i != 0) {
if (bareword.word.length < minTokenLength) {
barewords[i] = barewords[i - 1].join(bareword);
}
}
}
}
UnixTokenizer.tokenize({
required String input,
required int minTokenLength,
required int tokenLimit,
}) {
final matches = _tokenizePattern.allMatches(input);
final barewords = matches
.map((match) {
if (match.group(1) != null) {
return EnclosedBareword(match.group(1)!);
} else {
return SimpleBareword(match.group(2)!);
}
})
.where((token) => token.word.isNotEmpty)
.toList();
//Merge short tokens
_mergeShortBarewords(barewords, minTokenLength);
_tokens = barewords.take(tokenLimit).toList();
}
String build({bool wildcard});
}
final class UnixLikeQueryBuilder extends UnixTokenizer {
UnixLikeQueryBuilder.tokenize({
required super.input,
required super.minTokenLength,
required super.tokenLimit,
}) : super.tokenize();
@override
String build({bool wildcard = true}) {
final joined = _tokens.join('%');
return wildcard ? '%$joined%' : joined;
}
}
@@ -0,0 +1,143 @@
/*
* Copyright (c) 2024-2026 Fabian Freund.
*
* This file is part of WebLibre
* (see https://weblibre.eu).
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
typedef ToString<T> = String? Function(T);
class TokenizedFilter<T> {
final Iterable<T> original;
final List<T> filtered;
static List<String> _tokenize(String input) {
return input
.trim()
.toLowerCase()
.replaceAll(RegExp(r'[^\w\s]'), '') // Remove punctuation
.split(RegExp(r'\s+')) // Split on whitespace
.where((word) => word.isNotEmpty)
.toList();
}
static bool _matchTokens(
List<String> queryTokens,
List<String> targetTokens,
) {
return queryTokens.every(
(queryToken) =>
targetTokens.any((targetToken) => targetToken.contains(queryToken)),
);
}
static double _calculateScore(
List<String> queryTokens,
List<String> targetTokens,
) {
if (queryTokens.isEmpty || targetTokens.isEmpty) return 0.0;
var score = 0.0;
var exactMatches = 0;
var partialMatches = 0;
for (final queryToken in queryTokens) {
var hasMatch = false;
for (int i = 0; i < targetTokens.length; i++) {
final targetToken = targetTokens[i];
if (targetToken == queryToken) {
// Exact match gets 1.0 points
score += 1.0;
exactMatches++;
hasMatch = true;
break;
} else if (targetToken.contains(queryToken)) {
// Partial match gets points based on match length ratio
final ratio = queryToken.length / targetToken.length;
score += ratio * 0.8; // Partial matches worth 80% of exact matches
partialMatches++;
hasMatch = true;
break;
}
}
if (!hasMatch) {
score -= 0.2; // Penalty for unmatched query tokens
}
}
// Normalize score based on number of query tokens and matches
final matchRatio =
(exactMatches + partialMatches * 0.8) / queryTokens.length;
score = score / queryTokens.length; // Normalize by query length
// Apply match ratio as a multiplier
score *= matchRatio;
// Ensure score is between 0 and 1
return score.clamp(0.0, 1.0);
}
factory TokenizedFilter.sort({
required Iterable<T> items,
required ToString<T> toString,
required String query,
}) {
final queryTokens = _tokenize(query);
final scoredItems = items.map((item) {
final itemString = toString(item);
if (itemString == null || itemString.isEmpty) {
return (item, 0.0);
}
final itemTokens = _tokenize(itemString);
final score = _calculateScore(queryTokens, itemTokens);
return (item, score);
}).toList();
// Sort by score in descending order
scoredItems.sort((a, b) => b.$2.compareTo(a.$2));
// Extract just the items in sorted order
final sorted = scoredItems.map((e) => e.$1).toList();
return TokenizedFilter._(items, sorted);
}
factory TokenizedFilter.remove({
required Iterable<T> items,
required ToString<T> toString,
required String query,
}) {
final queryTokens = _tokenize(query);
final filtered = items.where((item) {
final itemString = toString(item);
if (itemString == null || itemString.isEmpty) {
return false;
}
final itemTokens = _tokenize(itemString);
return _matchTokens(queryTokens, itemTokens);
}).toList();
return TokenizedFilter._(items, filtered);
}
const TokenizedFilter._(this.original, this.filtered);
}