added popular sites module
This commit is contained in:
Fabian Freund
@@ -4,6 +4,9 @@
|
||||
*.pyc
|
||||
*.swp
|
||||
.DS_Store
|
||||
|
||||
# Build-time raw inputs for the popular-sites DB (only sites.db is committed)
|
||||
assets/sites/raw/
|
||||
.atom/
|
||||
.buildlog/
|
||||
.history
|
||||
@@ -31,6 +34,7 @@ migrate_working_dir/
|
||||
.pub/
|
||||
/build/
|
||||
assets/quotes/quotes.db
|
||||
assets/sites/sites.db
|
||||
assets/bangs/*
|
||||
!assets/bangs/
|
||||
!assets/bangs/weblibre_bangs.json
|
||||
|
||||
@@ -16,6 +16,7 @@ targets:
|
||||
databases:
|
||||
bangs: lib/features/bangs/data/database/database.dart
|
||||
quotes: lib/features/quotes/data/database/database.dart
|
||||
sites: lib/features/popular_sites/data/database/database.dart
|
||||
tabs: lib/features/geckoview/features/tabs/data/database/database.dart
|
||||
user: lib/features/user/data/database/database.dart
|
||||
web_feed: lib/features/web_feed/data/database/database.dart
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"_meta": {
|
||||
"description": "This file contains a serialized version of schema entities for drift.",
|
||||
"version": "1.3.0"
|
||||
},
|
||||
"options": {
|
||||
"store_date_time_values_as_text": false
|
||||
},
|
||||
"entities": [
|
||||
{
|
||||
"id": 0,
|
||||
"references": [],
|
||||
"type": "table",
|
||||
"data": {
|
||||
"name": "sites",
|
||||
"was_declared_in_moor": true,
|
||||
"columns": [
|
||||
{
|
||||
"name": "domain",
|
||||
"getter_name": "domain",
|
||||
"moor_type": "string",
|
||||
"nullable": false,
|
||||
"customConstraints": "NOT NULL PRIMARY KEY",
|
||||
"default_dart": null,
|
||||
"default_client_dart": null,
|
||||
"dsl_features": [
|
||||
"primary-key"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "rank",
|
||||
"getter_name": "rank",
|
||||
"moor_type": "int",
|
||||
"nullable": false,
|
||||
"customConstraints": "NOT NULL",
|
||||
"default_dart": null,
|
||||
"default_client_dart": null,
|
||||
"dsl_features": []
|
||||
}
|
||||
],
|
||||
"is_virtual": false,
|
||||
"without_rowid": false,
|
||||
"constraints": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"fixed_sql": [
|
||||
{
|
||||
"name": "sites",
|
||||
"sql": [
|
||||
{
|
||||
"dialect": "sqlite",
|
||||
"sql": "CREATE TABLE IF NOT EXISTS \"sites\" (\"domain\" TEXT NOT NULL PRIMARY KEY, \"rank\" INTEGER NOT NULL);"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
+1
-1
@@ -44,7 +44,7 @@ final class EngineSettingsReplicationServiceProvider
|
||||
}
|
||||
|
||||
String _$engineSettingsReplicationServiceHash() =>
|
||||
r'00439944023e8336dc879c70bb578120e221676d';
|
||||
r'a3d7628b7662ca8b828586e3fcf4e8925adbe062';
|
||||
|
||||
abstract class _$EngineSettingsReplicationService extends $Notifier<void> {
|
||||
void build();
|
||||
|
||||
+1
@@ -28,6 +28,7 @@ const _$SearchModuleTypeEnumMap = {
|
||||
SearchModuleType.history: 'history',
|
||||
SearchModuleType.localHistory: 'localHistory',
|
||||
SearchModuleType.combinedHistory: 'combinedHistory',
|
||||
SearchModuleType.popularSites: 'popularSites',
|
||||
SearchModuleType.historyHighlights: 'historyHighlights',
|
||||
SearchModuleType.topSites: 'topSites',
|
||||
SearchModuleType.recentHistory: 'recentHistory',
|
||||
|
||||
+11
-1
@@ -49,6 +49,13 @@ enum SearchModuleType {
|
||||
/// enabling [history] and [localHistory] separately.
|
||||
combinedHistory,
|
||||
|
||||
/// Popular-domain prefix completions from the bundled Tranco-derived
|
||||
/// `sites.db` asset (filtered against adult/gambling + tracker/CDN lists).
|
||||
/// Static popularity data, ranked below history and bookmarks so
|
||||
/// visited/saved sites always win. Domain autocomplete: typing "git"
|
||||
/// suggests github.com even with no local history.
|
||||
popularSites,
|
||||
|
||||
historyHighlights,
|
||||
topSites,
|
||||
recentHistory,
|
||||
@@ -67,6 +74,7 @@ enum SearchModuleType {
|
||||
history => 'History (engine)',
|
||||
localHistory => 'Local content',
|
||||
combinedHistory => 'History',
|
||||
popularSites => 'Popular Sites',
|
||||
historyHighlights => 'History Highlights',
|
||||
topSites => 'Top Sites',
|
||||
recentHistory => 'Recent History',
|
||||
@@ -100,6 +108,7 @@ enum SearchModuleGroup {
|
||||
SearchModuleType.bookmarks,
|
||||
SearchModuleType.articles,
|
||||
SearchModuleType.combinedHistory,
|
||||
SearchModuleType.popularSites,
|
||||
],
|
||||
);
|
||||
|
||||
@@ -125,7 +134,8 @@ extension SearchModuleTypeGroup on SearchModuleType {
|
||||
SearchModuleType.articles ||
|
||||
SearchModuleType.history ||
|
||||
SearchModuleType.localHistory ||
|
||||
SearchModuleType.combinedHistory => SearchModuleGroup.search,
|
||||
SearchModuleType.combinedHistory ||
|
||||
SearchModuleType.popularSites => SearchModuleGroup.search,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ import 'package:weblibre/features/geckoview/features/search/presentation/widgets
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/local_history_suggestions.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/popular_sites_suggestions.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/search_providers_section.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/search_term_suggestions_section.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/tab_search.dart';
|
||||
@@ -613,6 +614,10 @@ class SearchScreen extends HookConsumerWidget {
|
||||
searchTextListenable: sampledSearchText,
|
||||
onUriSelected: openUriInTab,
|
||||
),
|
||||
SearchModuleType.popularSites: PopularSitesSuggestions(
|
||||
searchTextListenable: sampledSearchText,
|
||||
onUriSelected: openUriInTab,
|
||||
),
|
||||
};
|
||||
|
||||
bool canShowSearchModule(SearchModuleType type) {
|
||||
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:hooks_riverpod/hooks_riverpod.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/domain/providers/search_modules_view.dart';
|
||||
import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/search_module_section.dart';
|
||||
import 'package:weblibre/features/popular_sites/domain/providers/popular_sites_search.dart';
|
||||
import 'package:weblibre/presentation/hooks/on_listenable_change_selector.dart';
|
||||
import 'package:weblibre/presentation/widgets/url_icon.dart';
|
||||
|
||||
/// Omnibar module offering popular-domain completions from the bundled
|
||||
/// Tranco-derived `sites.db`. Ranked below history and bookmarks so a user's
|
||||
/// own visited/saved sites always win; this fills the long tail with
|
||||
/// well-known destinations (typing "git" -> github.com) even with no history.
|
||||
class PopularSitesSuggestions extends HookConsumerWidget {
|
||||
final ValueListenable<TextEditingValue> searchTextListenable;
|
||||
final void Function(Uri uri) onUriSelected;
|
||||
|
||||
const PopularSitesSuggestions({
|
||||
super.key,
|
||||
required this.searchTextListenable,
|
||||
required this.onUriSelected,
|
||||
});
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context, WidgetRef ref) {
|
||||
final results = ref.watch(popularSitesSearchResultsProvider);
|
||||
|
||||
useOnListenableChangeSelector(
|
||||
searchTextListenable,
|
||||
() => searchTextListenable.value.text,
|
||||
() async {
|
||||
await ref
|
||||
.read(popularSitesSearchResultsProvider.notifier)
|
||||
.search(searchTextListenable.value.text);
|
||||
},
|
||||
);
|
||||
|
||||
if (results.isEmpty) {
|
||||
return const SliverToBoxAdapter(child: SizedBox.shrink());
|
||||
}
|
||||
|
||||
return SearchModuleSection(
|
||||
title: 'Popular Sites',
|
||||
moduleType: SearchModuleType.popularSites,
|
||||
totalCount: results.length,
|
||||
contentSliverBuilder:
|
||||
({required bool isCollapsed, required int visibleCount}) => [
|
||||
if (!isCollapsed)
|
||||
SliverList.builder(
|
||||
itemCount: visibleCount,
|
||||
itemBuilder: (context, index) {
|
||||
final site = results[index];
|
||||
final uri = Uri.parse('https://${site.domain}');
|
||||
|
||||
return ListTile(
|
||||
key: ValueKey(site.domain),
|
||||
leading: RepaintBoundary(
|
||||
child: UrlIcon([uri], iconSize: 24),
|
||||
),
|
||||
title: Text(
|
||||
site.domain,
|
||||
maxLines: 1,
|
||||
overflow: TextOverflow.ellipsis,
|
||||
),
|
||||
onTap: () => onUriSelected(uri),
|
||||
);
|
||||
},
|
||||
),
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/daos/site.drift.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/database.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart';
|
||||
|
||||
@DriftAccessor()
|
||||
class SiteDao extends DatabaseAccessor<SitesDatabase> with $SiteDaoMixin {
|
||||
SiteDao(super.db);
|
||||
|
||||
/// Popular-domain prefix completions for the omnibar, ordered by Tranco
|
||||
/// popularity (densely re-ranked at build time, so `rank` is a contiguous
|
||||
/// 1..N order). [prefix] is matched case-insensitively against the start of
|
||||
/// the registrable domain.
|
||||
///
|
||||
/// LIKE metacharacters (`%`, `_`, and the `\` escape char itself) in the
|
||||
/// user input are escaped so they match literally rather than acting as
|
||||
/// wildcards — see the `ESCAPE '\'` clause on `searchSitesByPrefix`.
|
||||
Selectable<Site> searchByPrefix(String prefix, {int limit = 8}) {
|
||||
final escaped = prefix
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replaceAll(r'\', r'\\')
|
||||
.replaceAll('%', r'\%')
|
||||
.replaceAll('_', r'\_');
|
||||
|
||||
return db.definitionsDrift.searchSitesByPrefix(
|
||||
pattern: '$escaped%',
|
||||
limit: limit,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// dart format width=80
|
||||
// ignore_for_file: type=lint
|
||||
import 'package:drift/drift.dart' as i0;
|
||||
import 'package:weblibre/features/popular_sites/data/database/database.dart'
|
||||
as i1;
|
||||
|
||||
mixin $SiteDaoMixin on i0.DatabaseAccessor<i1.SitesDatabase> {
|
||||
SiteDaoManager get managers => SiteDaoManager(this);
|
||||
}
|
||||
|
||||
class SiteDaoManager {
|
||||
final $SiteDaoMixin _db;
|
||||
SiteDaoManager(this._db);
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:drift_dev/api/migrations_native.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/daos/site.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/database.drift.dart';
|
||||
|
||||
@DriftDatabase(include: {'definitions.drift'}, daos: [SiteDao])
|
||||
class SitesDatabase extends $SitesDatabase {
|
||||
@override
|
||||
final int schemaVersion = 1;
|
||||
|
||||
@override
|
||||
MigrationStrategy get migration => MigrationStrategy(
|
||||
beforeOpen: (details) async {
|
||||
if (kDebugMode) {
|
||||
await validateDatabaseSchema();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
SitesDatabase(super.e);
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
// dart format width=80
|
||||
// ignore_for_file: type=lint
|
||||
import 'package:drift/drift.dart' as i0;
|
||||
import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart'
|
||||
as i1;
|
||||
import 'package:weblibre/features/popular_sites/data/database/daos/site.dart'
|
||||
as i2;
|
||||
import 'package:weblibre/features/popular_sites/data/database/database.dart'
|
||||
as i3;
|
||||
import 'package:drift/internal/modular.dart' as i4;
|
||||
import 'package:sqlite3/common.dart' as i5;
|
||||
|
||||
abstract class $SitesDatabase extends i0.GeneratedDatabase {
|
||||
$SitesDatabase(i0.QueryExecutor e) : super(e);
|
||||
$SitesDatabaseManager get managers => $SitesDatabaseManager(this);
|
||||
late final i1.Sites sites = i1.Sites(this);
|
||||
late final i2.SiteDao siteDao = i2.SiteDao(this as i3.SitesDatabase);
|
||||
i1.DefinitionsDrift get definitionsDrift => i4.ReadDatabaseContainer(
|
||||
this,
|
||||
).accessor<i1.DefinitionsDrift>(i1.DefinitionsDrift.new);
|
||||
@override
|
||||
Iterable<i0.TableInfo<i0.Table, Object?>> get allTables =>
|
||||
allSchemaEntities.whereType<i0.TableInfo<i0.Table, Object?>>();
|
||||
@override
|
||||
List<i0.DatabaseSchemaEntity> get allSchemaEntities => [sites];
|
||||
}
|
||||
|
||||
class $SitesDatabaseManager {
|
||||
final $SitesDatabase _db;
|
||||
$SitesDatabaseManager(this._db);
|
||||
i1.$SitesTableManager get sites => i1.$SitesTableManager(_db, _db.sites);
|
||||
}
|
||||
|
||||
extension DefineFunctions on i5.CommonDatabase {
|
||||
void defineFunctions({
|
||||
required String Function(int, String?) lexoRankNext,
|
||||
required String Function(int, String?) lexoRankPrevious,
|
||||
required String Function(String?, String?) lexoRankReorderAfter,
|
||||
required String Function(String?, String?) lexoRankReorderBefore,
|
||||
required int Function() generateContentHash,
|
||||
required bool Function(String?) urlIndexable,
|
||||
required String Function(String?) urlCanonical,
|
||||
required String Function(String?) urlHost,
|
||||
required String Function(String?) urlPath,
|
||||
}) {
|
||||
createFunction(
|
||||
functionName: 'lexo_rank_next',
|
||||
argumentCount: const i5.AllowedArgumentCount(2),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as int;
|
||||
final arg1 = args[1] as String?;
|
||||
return lexoRankNext(arg0, arg1);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'lexo_rank_previous',
|
||||
argumentCount: const i5.AllowedArgumentCount(2),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as int;
|
||||
final arg1 = args[1] as String?;
|
||||
return lexoRankPrevious(arg0, arg1);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'lexo_rank_reorder_after',
|
||||
argumentCount: const i5.AllowedArgumentCount(2),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as String?;
|
||||
final arg1 = args[1] as String?;
|
||||
return lexoRankReorderAfter(arg0, arg1);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'lexo_rank_reorder_before',
|
||||
argumentCount: const i5.AllowedArgumentCount(2),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as String?;
|
||||
final arg1 = args[1] as String?;
|
||||
return lexoRankReorderBefore(arg0, arg1);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'generate_content_hash',
|
||||
argumentCount: const i5.AllowedArgumentCount(0),
|
||||
function: (args) {
|
||||
return generateContentHash();
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'url_indexable',
|
||||
argumentCount: const i5.AllowedArgumentCount(1),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as String?;
|
||||
return urlIndexable(arg0);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'url_canonical',
|
||||
argumentCount: const i5.AllowedArgumentCount(1),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as String?;
|
||||
return urlCanonical(arg0);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'url_host',
|
||||
argumentCount: const i5.AllowedArgumentCount(1),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as String?;
|
||||
return urlHost(arg0);
|
||||
},
|
||||
);
|
||||
createFunction(
|
||||
functionName: 'url_path',
|
||||
argumentCount: const i5.AllowedArgumentCount(1),
|
||||
function: (args) {
|
||||
final arg0 = args[0] as String?;
|
||||
return urlPath(arg0);
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE sites (
|
||||
domain TEXT NOT NULL PRIMARY KEY,
|
||||
rank INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Prefix completion ordered by popularity. The caller escapes LIKE
|
||||
-- metacharacters in the user input and appends '%', so `ESCAPE '\'` makes any
|
||||
-- literal % / _ the user typed match literally instead of as wildcards.
|
||||
searchSitesByPrefix:
|
||||
SELECT * FROM sites
|
||||
WHERE domain LIKE :pattern ESCAPE '\'
|
||||
ORDER BY rank ASC
|
||||
LIMIT :limit;
|
||||
@@ -0,0 +1,334 @@
|
||||
// dart format width=80
|
||||
// ignore_for_file: type=lint
|
||||
import 'package:drift/drift.dart' as i0;
|
||||
import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart'
|
||||
as i1;
|
||||
import 'package:drift/internal/modular.dart' as i2;
|
||||
|
||||
typedef $SitesCreateCompanionBuilder =
|
||||
i1.SitesCompanion Function({
|
||||
required String domain,
|
||||
required int rank,
|
||||
i0.Value<int> rowid,
|
||||
});
|
||||
typedef $SitesUpdateCompanionBuilder =
|
||||
i1.SitesCompanion Function({
|
||||
i0.Value<String> domain,
|
||||
i0.Value<int> rank,
|
||||
i0.Value<int> rowid,
|
||||
});
|
||||
|
||||
class $SitesFilterComposer extends i0.Composer<i0.GeneratedDatabase, i1.Sites> {
|
||||
$SitesFilterComposer({
|
||||
required super.$db,
|
||||
required super.$table,
|
||||
super.joinBuilder,
|
||||
super.$addJoinBuilderToRootComposer,
|
||||
super.$removeJoinBuilderFromRootComposer,
|
||||
});
|
||||
i0.ColumnFilters<String> get domain => $composableBuilder(
|
||||
column: $table.domain,
|
||||
builder: (column) => i0.ColumnFilters(column),
|
||||
);
|
||||
|
||||
i0.ColumnFilters<int> get rank => $composableBuilder(
|
||||
column: $table.rank,
|
||||
builder: (column) => i0.ColumnFilters(column),
|
||||
);
|
||||
}
|
||||
|
||||
class $SitesOrderingComposer
|
||||
extends i0.Composer<i0.GeneratedDatabase, i1.Sites> {
|
||||
$SitesOrderingComposer({
|
||||
required super.$db,
|
||||
required super.$table,
|
||||
super.joinBuilder,
|
||||
super.$addJoinBuilderToRootComposer,
|
||||
super.$removeJoinBuilderFromRootComposer,
|
||||
});
|
||||
i0.ColumnOrderings<String> get domain => $composableBuilder(
|
||||
column: $table.domain,
|
||||
builder: (column) => i0.ColumnOrderings(column),
|
||||
);
|
||||
|
||||
i0.ColumnOrderings<int> get rank => $composableBuilder(
|
||||
column: $table.rank,
|
||||
builder: (column) => i0.ColumnOrderings(column),
|
||||
);
|
||||
}
|
||||
|
||||
class $SitesAnnotationComposer
|
||||
extends i0.Composer<i0.GeneratedDatabase, i1.Sites> {
|
||||
$SitesAnnotationComposer({
|
||||
required super.$db,
|
||||
required super.$table,
|
||||
super.joinBuilder,
|
||||
super.$addJoinBuilderToRootComposer,
|
||||
super.$removeJoinBuilderFromRootComposer,
|
||||
});
|
||||
i0.GeneratedColumn<String> get domain =>
|
||||
$composableBuilder(column: $table.domain, builder: (column) => column);
|
||||
|
||||
i0.GeneratedColumn<int> get rank =>
|
||||
$composableBuilder(column: $table.rank, builder: (column) => column);
|
||||
}
|
||||
|
||||
class $SitesTableManager
|
||||
extends
|
||||
i0.RootTableManager<
|
||||
i0.GeneratedDatabase,
|
||||
i1.Sites,
|
||||
i1.Site,
|
||||
i1.$SitesFilterComposer,
|
||||
i1.$SitesOrderingComposer,
|
||||
i1.$SitesAnnotationComposer,
|
||||
$SitesCreateCompanionBuilder,
|
||||
$SitesUpdateCompanionBuilder,
|
||||
(i1.Site, i0.BaseReferences<i0.GeneratedDatabase, i1.Sites, i1.Site>),
|
||||
i1.Site,
|
||||
i0.PrefetchHooks Function()
|
||||
> {
|
||||
$SitesTableManager(i0.GeneratedDatabase db, i1.Sites table)
|
||||
: super(
|
||||
i0.TableManagerState(
|
||||
db: db,
|
||||
table: table,
|
||||
createFilteringComposer: () =>
|
||||
i1.$SitesFilterComposer($db: db, $table: table),
|
||||
createOrderingComposer: () =>
|
||||
i1.$SitesOrderingComposer($db: db, $table: table),
|
||||
createComputedFieldComposer: () =>
|
||||
i1.$SitesAnnotationComposer($db: db, $table: table),
|
||||
updateCompanionCallback:
|
||||
({
|
||||
i0.Value<String> domain = const i0.Value.absent(),
|
||||
i0.Value<int> rank = const i0.Value.absent(),
|
||||
i0.Value<int> rowid = const i0.Value.absent(),
|
||||
}) => i1.SitesCompanion(domain: domain, rank: rank, rowid: rowid),
|
||||
createCompanionCallback:
|
||||
({
|
||||
required String domain,
|
||||
required int rank,
|
||||
i0.Value<int> rowid = const i0.Value.absent(),
|
||||
}) => i1.SitesCompanion.insert(
|
||||
domain: domain,
|
||||
rank: rank,
|
||||
rowid: rowid,
|
||||
),
|
||||
withReferenceMapper: (p0) => p0
|
||||
.map((e) => (e.readTable(table), i0.BaseReferences(db, table, e)))
|
||||
.toList(),
|
||||
prefetchHooksCallback: null,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
typedef $SitesProcessedTableManager =
|
||||
i0.ProcessedTableManager<
|
||||
i0.GeneratedDatabase,
|
||||
i1.Sites,
|
||||
i1.Site,
|
||||
i1.$SitesFilterComposer,
|
||||
i1.$SitesOrderingComposer,
|
||||
i1.$SitesAnnotationComposer,
|
||||
$SitesCreateCompanionBuilder,
|
||||
$SitesUpdateCompanionBuilder,
|
||||
(i1.Site, i0.BaseReferences<i0.GeneratedDatabase, i1.Sites, i1.Site>),
|
||||
i1.Site,
|
||||
i0.PrefetchHooks Function()
|
||||
>;
|
||||
|
||||
class Sites extends i0.Table with i0.TableInfo<Sites, i1.Site> {
|
||||
@override
|
||||
final i0.GeneratedDatabase attachedDatabase;
|
||||
final String? _alias;
|
||||
Sites(this.attachedDatabase, [this._alias]);
|
||||
late final i0.GeneratedColumn<String> domain = i0.GeneratedColumn<String>(
|
||||
'domain',
|
||||
aliasedName,
|
||||
false,
|
||||
type: i0.DriftSqlType.string,
|
||||
requiredDuringInsert: true,
|
||||
$customConstraints: 'NOT NULL PRIMARY KEY',
|
||||
);
|
||||
late final i0.GeneratedColumn<int> rank = i0.GeneratedColumn<int>(
|
||||
'rank',
|
||||
aliasedName,
|
||||
false,
|
||||
type: i0.DriftSqlType.int,
|
||||
requiredDuringInsert: true,
|
||||
$customConstraints: 'NOT NULL',
|
||||
);
|
||||
@override
|
||||
List<i0.GeneratedColumn> get $columns => [domain, rank];
|
||||
@override
|
||||
String get aliasedName => _alias ?? actualTableName;
|
||||
@override
|
||||
String get actualTableName => $name;
|
||||
static const String $name = 'sites';
|
||||
@override
|
||||
Set<i0.GeneratedColumn> get $primaryKey => {domain};
|
||||
@override
|
||||
i1.Site map(Map<String, dynamic> data, {String? tablePrefix}) {
|
||||
final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : '';
|
||||
return i1.Site(
|
||||
domain: attachedDatabase.typeMapping.read(
|
||||
i0.DriftSqlType.string,
|
||||
data['${effectivePrefix}domain'],
|
||||
)!,
|
||||
rank: attachedDatabase.typeMapping.read(
|
||||
i0.DriftSqlType.int,
|
||||
data['${effectivePrefix}rank'],
|
||||
)!,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Sites createAlias(String alias) {
|
||||
return Sites(attachedDatabase, alias);
|
||||
}
|
||||
|
||||
@override
|
||||
bool get dontWriteConstraints => true;
|
||||
}
|
||||
|
||||
class Site extends i0.DataClass implements i0.Insertable<i1.Site> {
|
||||
final String domain;
|
||||
final int rank;
|
||||
const Site({required this.domain, required this.rank});
|
||||
@override
|
||||
Map<String, i0.Expression> toColumns(bool nullToAbsent) {
|
||||
final map = <String, i0.Expression>{};
|
||||
map['domain'] = i0.Variable<String>(domain);
|
||||
map['rank'] = i0.Variable<int>(rank);
|
||||
return map;
|
||||
}
|
||||
|
||||
factory Site.fromJson(
|
||||
Map<String, dynamic> json, {
|
||||
i0.ValueSerializer? serializer,
|
||||
}) {
|
||||
serializer ??= i0.driftRuntimeOptions.defaultSerializer;
|
||||
return Site(
|
||||
domain: serializer.fromJson<String>(json['domain']),
|
||||
rank: serializer.fromJson<int>(json['rank']),
|
||||
);
|
||||
}
|
||||
@override
|
||||
Map<String, dynamic> toJson({i0.ValueSerializer? serializer}) {
|
||||
serializer ??= i0.driftRuntimeOptions.defaultSerializer;
|
||||
return <String, dynamic>{
|
||||
'domain': serializer.toJson<String>(domain),
|
||||
'rank': serializer.toJson<int>(rank),
|
||||
};
|
||||
}
|
||||
|
||||
i1.Site copyWith({String? domain, int? rank}) =>
|
||||
i1.Site(domain: domain ?? this.domain, rank: rank ?? this.rank);
|
||||
Site copyWithCompanion(i1.SitesCompanion data) {
|
||||
return Site(
|
||||
domain: data.domain.present ? data.domain.value : this.domain,
|
||||
rank: data.rank.present ? data.rank.value : this.rank,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return (StringBuffer('Site(')
|
||||
..write('domain: $domain, ')
|
||||
..write('rank: $rank')
|
||||
..write(')'))
|
||||
.toString();
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode => Object.hash(domain, rank);
|
||||
@override
|
||||
bool operator ==(Object other) =>
|
||||
identical(this, other) ||
|
||||
(other is i1.Site &&
|
||||
other.domain == this.domain &&
|
||||
other.rank == this.rank);
|
||||
}
|
||||
|
||||
class SitesCompanion extends i0.UpdateCompanion<i1.Site> {
|
||||
final i0.Value<String> domain;
|
||||
final i0.Value<int> rank;
|
||||
final i0.Value<int> rowid;
|
||||
const SitesCompanion({
|
||||
this.domain = const i0.Value.absent(),
|
||||
this.rank = const i0.Value.absent(),
|
||||
this.rowid = const i0.Value.absent(),
|
||||
});
|
||||
SitesCompanion.insert({
|
||||
required String domain,
|
||||
required int rank,
|
||||
this.rowid = const i0.Value.absent(),
|
||||
}) : domain = i0.Value(domain),
|
||||
rank = i0.Value(rank);
|
||||
static i0.Insertable<i1.Site> custom({
|
||||
i0.Expression<String>? domain,
|
||||
i0.Expression<int>? rank,
|
||||
i0.Expression<int>? rowid,
|
||||
}) {
|
||||
return i0.RawValuesInsertable({
|
||||
if (domain != null) 'domain': domain,
|
||||
if (rank != null) 'rank': rank,
|
||||
if (rowid != null) 'rowid': rowid,
|
||||
});
|
||||
}
|
||||
|
||||
i1.SitesCompanion copyWith({
|
||||
i0.Value<String>? domain,
|
||||
i0.Value<int>? rank,
|
||||
i0.Value<int>? rowid,
|
||||
}) {
|
||||
return i1.SitesCompanion(
|
||||
domain: domain ?? this.domain,
|
||||
rank: rank ?? this.rank,
|
||||
rowid: rowid ?? this.rowid,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Map<String, i0.Expression> toColumns(bool nullToAbsent) {
|
||||
final map = <String, i0.Expression>{};
|
||||
if (domain.present) {
|
||||
map['domain'] = i0.Variable<String>(domain.value);
|
||||
}
|
||||
if (rank.present) {
|
||||
map['rank'] = i0.Variable<int>(rank.value);
|
||||
}
|
||||
if (rowid.present) {
|
||||
map['rowid'] = i0.Variable<int>(rowid.value);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return (StringBuffer('SitesCompanion(')
|
||||
..write('domain: $domain, ')
|
||||
..write('rank: $rank, ')
|
||||
..write('rowid: $rowid')
|
||||
..write(')'))
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
|
||||
class DefinitionsDrift extends i2.ModularAccessor {
|
||||
DefinitionsDrift(i0.GeneratedDatabase db) : super(db);
|
||||
i0.Selectable<i1.Site> searchSitesByPrefix({
|
||||
required String pattern,
|
||||
required int limit,
|
||||
}) {
|
||||
return customSelect(
|
||||
'SELECT * FROM sites WHERE domain LIKE ?1 ESCAPE \'\\\' ORDER BY rank ASC LIMIT ?2',
|
||||
variables: [i0.Variable<String>(pattern), i0.Variable<int>(limit)],
|
||||
readsFrom: {sites},
|
||||
).asyncMap(sites.mapFromRow);
|
||||
}
|
||||
|
||||
i1.Sites get sites =>
|
||||
i2.ReadDatabaseContainer(attachedDatabase).resultSet<i1.Sites>('sites');
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:drift/drift.dart';
|
||||
import 'package:drift/native.dart';
|
||||
import 'package:flutter/services.dart' show rootBundle;
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:weblibre/core/database_registry.dart';
|
||||
import 'package:weblibre/core/filesystem.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/database.dart';
|
||||
|
||||
part 'providers.g.dart';
|
||||
|
||||
const _sitesAssetPath = 'assets/sites/sites.db';
|
||||
const _sitesDbFileName = 'sites.db';
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
SitesDatabase sitesDatabase(Ref ref) {
|
||||
final db = SitesDatabase(
|
||||
LazyDatabase(() async {
|
||||
final file = File(
|
||||
p.join(filesystem.profileDatabasesDir.path, _sitesDbFileName),
|
||||
);
|
||||
|
||||
await file.parent.create(recursive: true);
|
||||
|
||||
final blob = await rootBundle.load(_sitesAssetPath);
|
||||
final buffer = blob.buffer;
|
||||
await file.writeAsBytes(
|
||||
buffer.asUint8List(blob.offsetInBytes, blob.lengthInBytes),
|
||||
flush: true,
|
||||
);
|
||||
|
||||
return NativeDatabase.createInBackground(file);
|
||||
}),
|
||||
);
|
||||
|
||||
DatabaseRegistry.instance.register('sites', db);
|
||||
|
||||
ref.onDispose(() async {
|
||||
await db.close();
|
||||
});
|
||||
|
||||
return db;
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'providers.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
// ignore_for_file: type=lint, type=warning
|
||||
|
||||
@ProviderFor(sitesDatabase)
|
||||
final sitesDatabaseProvider = SitesDatabaseProvider._();
|
||||
|
||||
final class SitesDatabaseProvider
|
||||
extends $FunctionalProvider<SitesDatabase, SitesDatabase, SitesDatabase>
|
||||
with $Provider<SitesDatabase> {
|
||||
SitesDatabaseProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'sitesDatabaseProvider',
|
||||
isAutoDispose: false,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$sitesDatabaseHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
$ProviderElement<SitesDatabase> $createElement($ProviderPointer pointer) =>
|
||||
$ProviderElement(pointer);
|
||||
|
||||
@override
|
||||
SitesDatabase create(Ref ref) {
|
||||
return sitesDatabase(ref);
|
||||
}
|
||||
|
||||
/// {@macro riverpod.override_with_value}
|
||||
Override overrideWithValue(SitesDatabase value) {
|
||||
return $ProviderOverride(
|
||||
origin: this,
|
||||
providerOverride: $SyncValueProvider<SitesDatabase>(value),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
String _$sitesDatabaseHash() => r'ac11ab9c1da876c18460cd762237698c01b0ec62';
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/popular_sites/domain/repositories/popular_sites.dart';
|
||||
|
||||
part 'popular_sites_search.g.dart';
|
||||
|
||||
/// Holds the current popular-domain prefix completions for the omnibar
|
||||
/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers:
|
||||
/// the widget watches the state list and pushes new queries through [search].
|
||||
@Riverpod()
|
||||
class PopularSitesSearchResults extends _$PopularSitesSearchResults {
|
||||
Future<void> search(String query, {int limit = 8}) async {
|
||||
if (query.trim().isEmpty) {
|
||||
state = const [];
|
||||
return;
|
||||
}
|
||||
|
||||
final results = await ref
|
||||
.read(popularSitesRepositoryProvider.notifier)
|
||||
.searchByPrefix(query, limit: limit);
|
||||
|
||||
if (!ref.mounted) return;
|
||||
state = results;
|
||||
}
|
||||
|
||||
@override
|
||||
List<Site> build() {
|
||||
return const [];
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'popular_sites_search.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
// ignore_for_file: type=lint, type=warning
|
||||
/// Holds the current popular-domain prefix completions for the omnibar
|
||||
/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers:
|
||||
/// the widget watches the state list and pushes new queries through [search].
|
||||
|
||||
@ProviderFor(PopularSitesSearchResults)
|
||||
final popularSitesSearchResultsProvider = PopularSitesSearchResultsProvider._();
|
||||
|
||||
/// Holds the current popular-domain prefix completions for the omnibar
|
||||
/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers:
|
||||
/// the widget watches the state list and pushes new queries through [search].
|
||||
final class PopularSitesSearchResultsProvider
|
||||
extends $NotifierProvider<PopularSitesSearchResults, List<Site>> {
|
||||
/// Holds the current popular-domain prefix completions for the omnibar
|
||||
/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers:
|
||||
/// the widget watches the state list and pushes new queries through [search].
|
||||
PopularSitesSearchResultsProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'popularSitesSearchResultsProvider',
|
||||
isAutoDispose: true,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$popularSitesSearchResultsHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
PopularSitesSearchResults create() => PopularSitesSearchResults();
|
||||
|
||||
/// {@macro riverpod.override_with_value}
|
||||
Override overrideWithValue(List<Site> value) {
|
||||
return $ProviderOverride(
|
||||
origin: this,
|
||||
providerOverride: $SyncValueProvider<List<Site>>(value),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
String _$popularSitesSearchResultsHash() =>
|
||||
r'107396362aa70acf898b75137df42717ac52f1b3';
|
||||
|
||||
/// Holds the current popular-domain prefix completions for the omnibar
|
||||
/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers:
|
||||
/// the widget watches the state list and pushes new queries through [search].
|
||||
|
||||
abstract class _$PopularSitesSearchResults extends $Notifier<List<Site>> {
|
||||
List<Site> build();
|
||||
@$mustCallSuper
|
||||
@override
|
||||
WhenComplete runBuild() {
|
||||
final ref = this.ref as $Ref<List<Site>, List<Site>>;
|
||||
final element =
|
||||
ref.element
|
||||
as $ClassProviderElement<
|
||||
AnyNotifier<List<Site>, List<Site>>,
|
||||
List<Site>,
|
||||
Object?,
|
||||
Object?
|
||||
>;
|
||||
return element.handleCreate(ref, build);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright (c) 2024-2026 Fabian Freund.
|
||||
*
|
||||
* This file is part of WebLibre
|
||||
* (see https://weblibre.eu).
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
import 'package:riverpod_annotation/riverpod_annotation.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart';
|
||||
import 'package:weblibre/features/popular_sites/data/providers.dart';
|
||||
|
||||
part 'popular_sites.g.dart';
|
||||
|
||||
@Riverpod(keepAlive: true)
|
||||
class PopularSitesRepository extends _$PopularSitesRepository {
|
||||
/// Popular registrable domains whose start matches [prefix], ordered by
|
||||
/// popularity. Returns empty for a blank prefix so the omnibar suggestion
|
||||
/// module stays quiet until the user types.
|
||||
Future<List<Site>> searchByPrefix(String prefix, {int limit = 8}) {
|
||||
if (prefix.trim().isEmpty) {
|
||||
return Future.value(const []);
|
||||
}
|
||||
|
||||
return ref
|
||||
.read(sitesDatabaseProvider)
|
||||
.siteDao
|
||||
.searchByPrefix(prefix, limit: limit)
|
||||
.get();
|
||||
}
|
||||
|
||||
@override
|
||||
void build() {}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
|
||||
part of 'popular_sites.dart';
|
||||
|
||||
// **************************************************************************
|
||||
// RiverpodGenerator
|
||||
// **************************************************************************
|
||||
|
||||
// GENERATED CODE - DO NOT MODIFY BY HAND
|
||||
// ignore_for_file: type=lint, type=warning
|
||||
|
||||
@ProviderFor(PopularSitesRepository)
|
||||
final popularSitesRepositoryProvider = PopularSitesRepositoryProvider._();
|
||||
|
||||
final class PopularSitesRepositoryProvider
|
||||
extends $NotifierProvider<PopularSitesRepository, void> {
|
||||
PopularSitesRepositoryProvider._()
|
||||
: super(
|
||||
from: null,
|
||||
argument: null,
|
||||
retry: null,
|
||||
name: r'popularSitesRepositoryProvider',
|
||||
isAutoDispose: false,
|
||||
dependencies: null,
|
||||
$allTransitiveDependencies: null,
|
||||
);
|
||||
|
||||
@override
|
||||
String debugGetCreateSourceHash() => _$popularSitesRepositoryHash();
|
||||
|
||||
@$internal
|
||||
@override
|
||||
PopularSitesRepository create() => PopularSitesRepository();
|
||||
|
||||
/// {@macro riverpod.override_with_value}
|
||||
Override overrideWithValue(void value) {
|
||||
return $ProviderOverride(
|
||||
origin: this,
|
||||
providerOverride: $SyncValueProvider<void>(value),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
String _$popularSitesRepositoryHash() =>
|
||||
r'9825dc634b2ae9ea48a33eac8387a3e138c17b9f';
|
||||
|
||||
abstract class _$PopularSitesRepository extends $Notifier<void> {
|
||||
void build();
|
||||
@$mustCallSuper
|
||||
@override
|
||||
WhenComplete runBuild() {
|
||||
final ref = this.ref as $Ref<void, void>;
|
||||
final element =
|
||||
ref.element
|
||||
as $ClassProviderElement<
|
||||
AnyNotifier<void, void>,
|
||||
void,
|
||||
Object?,
|
||||
Object?
|
||||
>;
|
||||
return element.handleCreate(ref, build);
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,7 @@ dependencies:
|
||||
fading_scroll: ^0.9.4
|
||||
fancy_password_field: ^2.0.8
|
||||
fast_equatable: ^1.3.1
|
||||
file_picker: ^12.0.0-beta.7
|
||||
file_picker: 12.0.0-beta.5
|
||||
flutter:
|
||||
sdk: flutter
|
||||
flutter_auto_size_text: ^5.0.0
|
||||
@@ -141,6 +141,7 @@ flutter:
|
||||
- assets/preferences/
|
||||
- assets/legal/
|
||||
- assets/quotes/
|
||||
- assets/sites/
|
||||
- assets/small_web/
|
||||
- assets/ublock/
|
||||
fonts:
|
||||
|
||||
+2
-1
@@ -73,12 +73,13 @@ melos:
|
||||
description: Download external assets (bangs, bridges, URL lists)
|
||||
run: bash scripts/update-assets.sh
|
||||
build-components:
|
||||
description: Build JS components and generate quotes DB
|
||||
description: Build JS components and generate quotes/sites DBs
|
||||
run: |
|
||||
set -e
|
||||
sh scripts/build-container-proxy.sh
|
||||
sh scripts/build-readability.sh
|
||||
python3 scripts/build_quotes_db.py
|
||||
python3 scripts/build_sites_db.py
|
||||
build-singbox-libbox:
|
||||
description: Build official sing-box Android libbox AAR for flutter_singbox_proxy
|
||||
run: packages/flutter_singbox_proxy/scripts/build-libbox-android.sh
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build a SQLite database of popular websites for omnibar autocomplete.
|
||||
|
||||
Source of truth is the Tranco top-1M ranking (manipulation-resistant aggregate
|
||||
of Umbrella/Majestic/Cloudflare/Farsight). Tranco ranks *domains by traffic*,
|
||||
not "sites a human would open", so the head is polluted with adult/gambling
|
||||
sites, ad/tracker endpoints and pure CDN/infra domains. We filter those out
|
||||
using:
|
||||
|
||||
* StevenBlack "gambling-porn-only" hosts -> adult + gambling
|
||||
* Disconnect services.json (Advertising / Analytics / FingerprintingInvasive
|
||||
/ Cryptomining categories only -- NOT Social/Content, which contain
|
||||
first-party sites people actually visit) -> ad/tracker/fingerprint domains
|
||||
* a small static denylist of well-known CDN / cloud / API registrable
|
||||
domains that the tracker lists don't classify
|
||||
|
||||
The survivors (first --limit by rank) are written to a single read-only table
|
||||
consumed as a bundled asset, mirroring scripts/build_quotes_db.py.
|
||||
|
||||
Raw inputs are produced by `scripts/update-assets.sh --group popular-sites`.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
RAW_DIR = REPO_ROOT / "apps" / "weblibre" / "assets" / "sites" / "raw"
|
||||
DEFAULT_TRANCO = RAW_DIR / "tranco-top-1m.csv.zip"
|
||||
DEFAULT_BLOCKLIST = RAW_DIR / "stevenblack-gambling-porn.txt"
|
||||
DEFAULT_DISCONNECT = RAW_DIR / "disconnect-services.json"
|
||||
DEFAULT_OUTPUT = REPO_ROOT / "apps" / "weblibre" / "assets" / "sites" / "sites.db"
|
||||
DEFAULT_TABLE = "sites"
|
||||
|
||||
# How many top-ranked Tranco domains to consider before filtering. Filtering
|
||||
# removes a sizeable chunk of the head, so we scan well past --limit to still
|
||||
# fill the quota with genuinely popular sites.
|
||||
DEFAULT_SCAN_LIMIT = 250_000
|
||||
# Final number of clean domains to keep.
|
||||
DEFAULT_LIMIT = 25_000
|
||||
|
||||
# Disconnect categories to treat as noise. Social/Content/Email are excluded on
|
||||
# purpose: they list first-party destinations (facebook.com, twitter.com, ...)
|
||||
# that users legitimately want autocompleted.
|
||||
DISCONNECT_BLOCK_CATEGORIES = (
|
||||
"Advertising",
|
||||
"Analytics",
|
||||
"FingerprintingInvasive",
|
||||
"Cryptomining",
|
||||
)
|
||||
|
||||
# Pure CDN / cloud / DNS / registrar / API registrable domains that the tracker
|
||||
# lists generally don't flag but which are never a navigation target. The
|
||||
# Disconnect list is great for *trackers* but does not enumerate this infra
|
||||
# (verified: gtld-servers.net, domaincontrol.com, googletagmanager.com,
|
||||
# appsflyersdk.com aren't in it at all; googlevideo.com is only under the
|
||||
# Content category we keep). Kept deliberately famous; the `cdn`-substring rule
|
||||
# below sweeps the generic-CDN long tail.
|
||||
STATIC_INFRA_DENYLIST = frozenset(
|
||||
{
|
||||
# CDN / cloud edge
|
||||
"fbcdn.net",
|
||||
"gstatic.com",
|
||||
"googleusercontent.com",
|
||||
"ggpht.com",
|
||||
"googleapis.com",
|
||||
"googlevideo.com",
|
||||
"gvt1.com",
|
||||
"gvt2.com",
|
||||
"akamai.net",
|
||||
"akamaihd.net",
|
||||
"akamaiedge.net",
|
||||
"akamaized.net",
|
||||
"akadns.net",
|
||||
"edgekey.net",
|
||||
"edgesuite.net",
|
||||
"cloudfront.net",
|
||||
"amazonaws.com",
|
||||
"azureedge.net",
|
||||
"windows.net",
|
||||
"trafficmanager.net",
|
||||
"fastly.net",
|
||||
"fastlylb.net",
|
||||
"llnwd.net",
|
||||
"stackpathdns.com",
|
||||
"aaplimg.com",
|
||||
"apple-dns.net",
|
||||
# DNS / registry / registrar infra
|
||||
"gtld-servers.net",
|
||||
"nstld.com",
|
||||
"domaincontrol.com",
|
||||
"ripn.net",
|
||||
# App SDK / measurement endpoints Disconnect misses
|
||||
"app-measurement.com",
|
||||
"googletagmanager.com",
|
||||
"appsflyersdk.com",
|
||||
# Device / IoT cloud phone-home (high DNS volume, not navigable)
|
||||
"ezviz7.com",
|
||||
"hicloudcam.com",
|
||||
"whatsapp.net",
|
||||
}
|
||||
)
|
||||
|
||||
# Drop any registrable domain containing this token. Verified against the kept
|
||||
# set: every match is a CDN backend (tiktokcdn, alicdn, spotifycdn, licdn,
|
||||
# b-cdn, ...) with no first-party site among them, so a plain substring test is
|
||||
# safe and avoids a broad multi-pattern infra regex.
|
||||
INFRA_SUBSTRINGS = ("cdn",)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Build a SQLite popular-sites database from a Tranco list."
|
||||
)
|
||||
parser.add_argument("--tranco", default=str(DEFAULT_TRANCO))
|
||||
parser.add_argument("--blocklist", default=str(DEFAULT_BLOCKLIST))
|
||||
parser.add_argument("--disconnect", default=str(DEFAULT_DISCONNECT))
|
||||
parser.add_argument("--output", default=str(DEFAULT_OUTPUT))
|
||||
parser.add_argument("--table", default=DEFAULT_TABLE)
|
||||
parser.add_argument("--scan-limit", type=int, default=DEFAULT_SCAN_LIMIT)
|
||||
parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_tranco(path: Path, scan_limit: int) -> list[tuple[int, str]]:
|
||||
"""Return [(rank, domain)] for the first `scan_limit` Tranco rows.
|
||||
|
||||
Accepts either the raw `rank,domain` CSV or the `.zip` it ships in.
|
||||
"""
|
||||
if path.suffix == ".zip":
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
csv_name = next(n for n in archive.namelist() if n.endswith(".csv"))
|
||||
raw = archive.read(csv_name)
|
||||
handle = io.TextIOWrapper(io.BytesIO(raw), encoding="utf-8")
|
||||
else:
|
||||
handle = path.open("r", encoding="utf-8")
|
||||
|
||||
rows: list[tuple[int, str]] = []
|
||||
with handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rank_str, _, domain = line.partition(",")
|
||||
domain = domain.strip().lower()
|
||||
if not domain:
|
||||
continue
|
||||
try:
|
||||
rank = int(rank_str)
|
||||
except ValueError:
|
||||
continue
|
||||
rows.append((rank, domain))
|
||||
if len(rows) >= scan_limit:
|
||||
break
|
||||
return rows
|
||||
|
||||
|
||||
def load_hosts_domains(path: Path) -> set[str]:
|
||||
"""Parse a hosts-format file (`0.0.0.0 domain`) into a domain set."""
|
||||
domains: set[str] = set()
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
domain = parts[1].strip().lower()
|
||||
if domain and domain not in ("localhost", "localhost.localdomain"):
|
||||
domains.add(domain)
|
||||
return domains
|
||||
|
||||
|
||||
def load_disconnect_domains(path: Path, categories: tuple[str, ...]) -> set[str]:
|
||||
"""Collect tracker domains from the selected Disconnect categories.
|
||||
|
||||
Structure: categories -> [ {Company: {homepage_url: [domain, ...]}}, ... ].
|
||||
Non-list property values (metadata flags) are skipped.
|
||||
"""
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
|
||||
domains: set[str] = set()
|
||||
all_categories = payload.get("categories", {})
|
||||
for category in categories:
|
||||
for entry in all_categories.get(category, []):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for company_props in entry.values():
|
||||
if not isinstance(company_props, dict):
|
||||
continue
|
||||
for value in company_props.values():
|
||||
if isinstance(value, list):
|
||||
domains.update(d.strip().lower() for d in value if d)
|
||||
return domains
|
||||
|
||||
|
||||
def build_denylist(
|
||||
blocklist_path: Path, disconnect_path: Path
|
||||
) -> set[str]:
|
||||
deny = set(STATIC_INFRA_DENYLIST)
|
||||
deny |= load_hosts_domains(blocklist_path)
|
||||
deny |= load_disconnect_domains(disconnect_path, DISCONNECT_BLOCK_CATEGORIES)
|
||||
return deny
|
||||
|
||||
|
||||
def filter_domains(
|
||||
tranco: list[tuple[int, str]], deny: set[str], limit: int
|
||||
) -> tuple[list[tuple[str, int]], int]:
|
||||
"""Keep ranked domains not in the denylist, up to `limit`.
|
||||
|
||||
Re-ranks survivors densely (1..N) so the stored rank is a contiguous
|
||||
popularity order for `ORDER BY rank` queries. Returns `(kept, scanned)`
|
||||
where `scanned` is how many Tranco rows were consumed to fill the quota,
|
||||
so the caller can report the real drop rate within the scanned head.
|
||||
"""
|
||||
kept: list[tuple[str, int]] = []
|
||||
new_rank = 0
|
||||
scanned = 0
|
||||
for _, domain in tranco:
|
||||
scanned += 1
|
||||
if domain in deny:
|
||||
continue
|
||||
if any(token in domain for token in INFRA_SUBSTRINGS):
|
||||
continue
|
||||
new_rank += 1
|
||||
kept.append((domain, new_rank))
|
||||
if len(kept) >= limit:
|
||||
break
|
||||
return kept, scanned
|
||||
|
||||
|
||||
def quote_identifier(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def build_database(
|
||||
db_path: Path, table_name: str, rows: list[tuple[str, int]]
|
||||
) -> None:
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
table = quote_identifier(table_name)
|
||||
|
||||
connection = sqlite3.connect(db_path)
|
||||
try:
|
||||
cursor = connection.cursor()
|
||||
cursor.execute(f"DROP TABLE IF EXISTS {table}")
|
||||
cursor.execute(
|
||||
f"""
|
||||
CREATE TABLE {table} (
|
||||
domain TEXT NOT NULL PRIMARY KEY,
|
||||
rank INTEGER NOT NULL
|
||||
)
|
||||
"""
|
||||
)
|
||||
cursor.executemany(
|
||||
f"INSERT OR IGNORE INTO {table} (domain, rank) VALUES (?, ?)",
|
||||
rows,
|
||||
)
|
||||
connection.commit()
|
||||
cursor.execute("VACUUM")
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
tranco_path = Path(args.tranco).expanduser().resolve()
|
||||
blocklist_path = Path(args.blocklist).expanduser().resolve()
|
||||
disconnect_path = Path(args.disconnect).expanduser().resolve()
|
||||
db_path = Path(args.output).expanduser().resolve()
|
||||
|
||||
# The raw inputs live under assets/sites/raw/ and are gitignored — they are
|
||||
# produced by `melos run update-assets`. A clean checkout has the committed
|
||||
# sites.db but not the raw inputs, so `melos run build-components` must not
|
||||
# fail there: skip rebuilding and keep the committed artifact. Only error if
|
||||
# there is no committed DB to fall back on.
|
||||
missing = [p for p in (tranco_path, blocklist_path, disconnect_path) if not p.exists()]
|
||||
if missing:
|
||||
names = ", ".join(p.name for p in missing)
|
||||
if db_path.exists():
|
||||
print(
|
||||
f"Skipping sites.db rebuild: missing raw input(s) [{names}]. "
|
||||
f"Keeping committed {db_path}. Run "
|
||||
f"`melos run update-assets` to refresh from source."
|
||||
)
|
||||
return
|
||||
raise SystemExit(
|
||||
f"Cannot build {db_path}: missing raw input(s) [{names}] and no "
|
||||
f"committed DB to fall back on. Run "
|
||||
f"`melos run update-assets --no-select` first."
|
||||
)
|
||||
|
||||
tranco = load_tranco(tranco_path, args.scan_limit)
|
||||
deny = build_denylist(blocklist_path, disconnect_path)
|
||||
rows, scanned = filter_domains(tranco, deny, args.limit)
|
||||
|
||||
build_database(db_path, args.table, rows)
|
||||
|
||||
dropped = scanned - len(rows)
|
||||
drop_pct = (dropped / scanned * 100) if scanned else 0
|
||||
print(
|
||||
f"Denylist={len(deny)} domains. Scanned top {scanned} Tranco ranks to "
|
||||
f"keep {len(rows)} sites (dropped {dropped}, {drop_pct:.0f}% of head). "
|
||||
f"Wrote {db_path}."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -104,9 +104,36 @@ update_ublock() {
|
||||
log "uBlock assets sync completed at $(cat "$dir/last_sync.txt")"
|
||||
}
|
||||
|
||||
# Raw inputs for the popular-sites database. These are build-time only and are
|
||||
# consumed by scripts/build_sites_db.py to produce assets/sites/sites.db. The
|
||||
# raw/ directory is gitignored; only the compiled sites.db is committed.
|
||||
update_popular_sites() {
|
||||
local dir="$REPO_ROOT/apps/weblibre/assets/sites/raw"
|
||||
log "Updating popular-sites source lists..."
|
||||
|
||||
# Tranco top-1M (manipulation-resistant aggregate ranking). The download
|
||||
# endpoint 307-redirects to the latest list; fetch() follows redirects.
|
||||
fetch "https://tranco-list.eu/top-1m.csv.zip" \
|
||||
"$dir/tranco-top-1m.csv.zip"
|
||||
|
||||
# StevenBlack "gambling-porn-only": the porn + gambling category domains
|
||||
# WITHOUT the unified malware/ad base list mixed in.
|
||||
fetch "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts" \
|
||||
"$dir/stevenblack-gambling-porn.txt"
|
||||
|
||||
# Disconnect tracker list (basis of Firefox ETP). build_sites_db.py pulls
|
||||
# only the Advertising/Analytics/FingerprintingInvasive/Cryptomining
|
||||
# categories so first-party Social/Content sites are preserved.
|
||||
fetch "https://raw.githubusercontent.com/disconnectme/disconnect-tracking-protection/master/services.json" \
|
||||
"$dir/disconnect-services.json"
|
||||
|
||||
date -u --iso-8601=seconds > "$dir/last_sync.txt"
|
||||
log "Popular-sites sources sync completed at $(cat "$dir/last_sync.txt")"
|
||||
}
|
||||
|
||||
# ── main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
ALL_GROUPS=(bangs bridges url-cleaner url-shorteners ublock)
|
||||
ALL_GROUPS=(bangs bridges url-cleaner url-shorteners ublock popular-sites)
|
||||
SELECTED_GROUPS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
@@ -129,6 +156,7 @@ for group in "${SELECTED_GROUPS[@]}"; do
|
||||
url-cleaner) update_url_cleaner || ((FAILURES++)) ;;
|
||||
url-shorteners) update_url_shorteners || ((FAILURES++)) ;;
|
||||
ublock) update_ublock || ((FAILURES++)) ;;
|
||||
popular-sites) update_popular_sites || ((FAILURES++)) ;;
|
||||
*) err "Unknown group: $group"; ((FAILURES++)) ;;
|
||||
esac
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user