From a1ed66b3921710e97f9edfbe6f22fc05fc97ad50 Mon Sep 17 00:00:00 2001 From: Fabian Freund Date: Mon, 29 Jun 2026 08:55:43 +0200 Subject: [PATCH] added popular sites module --- apps/weblibre/.gitignore | 4 + apps/weblibre/build.yaml | 1 + .../drift_schemas/sites/drift_schema_v1.json | 58 +++ .../engine_settings_replication.g.dart | 2 +- .../providers/search_module_order.g.dart | 1 + .../domain/providers/search_modules_view.dart | 12 +- .../search/presentation/screens/search.dart | 5 + .../popular_sites_suggestions.dart | 91 +++++ .../data/database/daos/site.dart | 50 +++ .../data/database/daos/site.drift.dart | 14 + .../popular_sites/data/database/database.dart | 41 +++ .../data/database/database.drift.dart | 122 +++++++ .../data/database/definitions.drift | 13 + .../data/database/definitions.drift.dart | 334 ++++++++++++++++++ .../popular_sites/data/providers.dart | 64 ++++ .../popular_sites/data/providers.g.dart | 51 +++ .../providers/popular_sites_search.dart | 49 +++ .../providers/popular_sites_search.g.dart | 76 ++++ .../domain/repositories/popular_sites.dart | 45 +++ .../domain/repositories/popular_sites.g.dart | 63 ++++ apps/weblibre/pubspec.yaml | 3 +- pubspec.yaml | 3 +- scripts/build_sites_db.py | 314 ++++++++++++++++ scripts/update-assets.sh | 30 +- 24 files changed, 1441 insertions(+), 5 deletions(-) create mode 100644 apps/weblibre/drift_schemas/sites/drift_schema_v1.json create mode 100644 apps/weblibre/lib/features/geckoview/features/search/presentation/widgets/search_modules/popular_sites_suggestions.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/database/daos/site.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/database/daos/site.drift.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/database/database.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/database/database.drift.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/database/definitions.drift create mode 100644 apps/weblibre/lib/features/popular_sites/data/database/definitions.drift.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/providers.dart create mode 100644 apps/weblibre/lib/features/popular_sites/data/providers.g.dart create mode 100644 apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.dart create mode 100644 apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.g.dart create mode 100644 apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.dart create mode 100644 apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.g.dart create mode 100644 scripts/build_sites_db.py diff --git a/apps/weblibre/.gitignore b/apps/weblibre/.gitignore index a0b27641..fc068b94 100644 --- a/apps/weblibre/.gitignore +++ b/apps/weblibre/.gitignore @@ -4,6 +4,9 @@ *.pyc *.swp .DS_Store + +# Build-time raw inputs for the popular-sites DB (only sites.db is committed) +assets/sites/raw/ .atom/ .buildlog/ .history @@ -31,6 +34,7 @@ migrate_working_dir/ .pub/ /build/ assets/quotes/quotes.db +assets/sites/sites.db assets/bangs/* !assets/bangs/ !assets/bangs/weblibre_bangs.json diff --git a/apps/weblibre/build.yaml b/apps/weblibre/build.yaml index dcd81b0d..9cbfbce2 100644 --- a/apps/weblibre/build.yaml +++ b/apps/weblibre/build.yaml @@ -16,6 +16,7 @@ targets: databases: bangs: lib/features/bangs/data/database/database.dart quotes: lib/features/quotes/data/database/database.dart + sites: lib/features/popular_sites/data/database/database.dart tabs: lib/features/geckoview/features/tabs/data/database/database.dart user: lib/features/user/data/database/database.dart web_feed: lib/features/web_feed/data/database/database.dart diff --git a/apps/weblibre/drift_schemas/sites/drift_schema_v1.json b/apps/weblibre/drift_schemas/sites/drift_schema_v1.json new file mode 100644 index 00000000..666e4161 --- /dev/null +++ b/apps/weblibre/drift_schemas/sites/drift_schema_v1.json @@ -0,0 +1,58 @@ +{ + "_meta": { + "description": "This file contains a serialized version of schema entities for drift.", + "version": "1.3.0" + }, + "options": { + "store_date_time_values_as_text": false + }, + "entities": [ + { + "id": 0, + "references": [], + "type": "table", + "data": { + "name": "sites", + "was_declared_in_moor": true, + "columns": [ + { + "name": "domain", + "getter_name": "domain", + "moor_type": "string", + "nullable": false, + "customConstraints": "NOT NULL PRIMARY KEY", + "default_dart": null, + "default_client_dart": null, + "dsl_features": [ + "primary-key" + ] + }, + { + "name": "rank", + "getter_name": "rank", + "moor_type": "int", + "nullable": false, + "customConstraints": "NOT NULL", + "default_dart": null, + "default_client_dart": null, + "dsl_features": [] + } + ], + "is_virtual": false, + "without_rowid": false, + "constraints": [] + } + } + ], + "fixed_sql": [ + { + "name": "sites", + "sql": [ + { + "dialect": "sqlite", + "sql": "CREATE TABLE IF NOT EXISTS \"sites\" (\"domain\" TEXT NOT NULL PRIMARY KEY, \"rank\" INTEGER NOT NULL);" + } + ] + } + ] +} \ No newline at end of file diff --git a/apps/weblibre/lib/features/geckoview/features/browser/domain/services/engine_settings_replication.g.dart b/apps/weblibre/lib/features/geckoview/features/browser/domain/services/engine_settings_replication.g.dart index e1618cfd..5950da74 100644 --- a/apps/weblibre/lib/features/geckoview/features/browser/domain/services/engine_settings_replication.g.dart +++ b/apps/weblibre/lib/features/geckoview/features/browser/domain/services/engine_settings_replication.g.dart @@ -44,7 +44,7 @@ final class EngineSettingsReplicationServiceProvider } String _$engineSettingsReplicationServiceHash() => - r'00439944023e8336dc879c70bb578120e221676d'; + r'a3d7628b7662ca8b828586e3fcf4e8925adbe062'; abstract class _$EngineSettingsReplicationService extends $Notifier { void build(); diff --git a/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_module_order.g.dart b/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_module_order.g.dart index 975cbfe3..254aee05 100644 --- a/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_module_order.g.dart +++ b/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_module_order.g.dart @@ -28,6 +28,7 @@ const _$SearchModuleTypeEnumMap = { SearchModuleType.history: 'history', SearchModuleType.localHistory: 'localHistory', SearchModuleType.combinedHistory: 'combinedHistory', + SearchModuleType.popularSites: 'popularSites', SearchModuleType.historyHighlights: 'historyHighlights', SearchModuleType.topSites: 'topSites', SearchModuleType.recentHistory: 'recentHistory', diff --git a/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_modules_view.dart b/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_modules_view.dart index 7f956ab1..7af6f309 100644 --- a/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_modules_view.dart +++ b/apps/weblibre/lib/features/geckoview/features/search/domain/providers/search_modules_view.dart @@ -49,6 +49,13 @@ enum SearchModuleType { /// enabling [history] and [localHistory] separately. combinedHistory, + /// Popular-domain prefix completions from the bundled Tranco-derived + /// `sites.db` asset (filtered against adult/gambling + tracker/CDN lists). + /// Static popularity data, ranked below history and bookmarks so + /// visited/saved sites always win. Domain autocomplete: typing "git" + /// suggests github.com even with no local history. + popularSites, + historyHighlights, topSites, recentHistory, @@ -67,6 +74,7 @@ enum SearchModuleType { history => 'History (engine)', localHistory => 'Local content', combinedHistory => 'History', + popularSites => 'Popular Sites', historyHighlights => 'History Highlights', topSites => 'Top Sites', recentHistory => 'Recent History', @@ -100,6 +108,7 @@ enum SearchModuleGroup { SearchModuleType.bookmarks, SearchModuleType.articles, SearchModuleType.combinedHistory, + SearchModuleType.popularSites, ], ); @@ -125,7 +134,8 @@ extension SearchModuleTypeGroup on SearchModuleType { SearchModuleType.articles || SearchModuleType.history || SearchModuleType.localHistory || - SearchModuleType.combinedHistory => SearchModuleGroup.search, + SearchModuleType.combinedHistory || + SearchModuleType.popularSites => SearchModuleGroup.search, }; } diff --git a/apps/weblibre/lib/features/geckoview/features/search/presentation/screens/search.dart b/apps/weblibre/lib/features/geckoview/features/search/presentation/screens/search.dart index ef90bda0..6456926a 100644 --- a/apps/weblibre/lib/features/geckoview/features/search/presentation/screens/search.dart +++ b/apps/weblibre/lib/features/geckoview/features/search/presentation/screens/search.dart @@ -59,6 +59,7 @@ import 'package:weblibre/features/geckoview/features/search/presentation/widgets import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/feed_search.dart'; import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/history_suggestions.dart'; import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/local_history_suggestions.dart'; +import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/popular_sites_suggestions.dart'; import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/search_providers_section.dart'; import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/search_term_suggestions_section.dart'; import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/tab_search.dart'; @@ -613,6 +614,10 @@ class SearchScreen extends HookConsumerWidget { searchTextListenable: sampledSearchText, onUriSelected: openUriInTab, ), + SearchModuleType.popularSites: PopularSitesSuggestions( + searchTextListenable: sampledSearchText, + onUriSelected: openUriInTab, + ), }; bool canShowSearchModule(SearchModuleType type) { diff --git a/apps/weblibre/lib/features/geckoview/features/search/presentation/widgets/search_modules/popular_sites_suggestions.dart b/apps/weblibre/lib/features/geckoview/features/search/presentation/widgets/search_modules/popular_sites_suggestions.dart new file mode 100644 index 00000000..ff3fd3f7 --- /dev/null +++ b/apps/weblibre/lib/features/geckoview/features/search/presentation/widgets/search_modules/popular_sites_suggestions.dart @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2024-2026 Fabian Freund. + * + * This file is part of WebLibre + * (see https://weblibre.eu). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:weblibre/features/geckoview/features/search/domain/providers/search_modules_view.dart'; +import 'package:weblibre/features/geckoview/features/search/presentation/widgets/search_modules/search_module_section.dart'; +import 'package:weblibre/features/popular_sites/domain/providers/popular_sites_search.dart'; +import 'package:weblibre/presentation/hooks/on_listenable_change_selector.dart'; +import 'package:weblibre/presentation/widgets/url_icon.dart'; + +/// Omnibar module offering popular-domain completions from the bundled +/// Tranco-derived `sites.db`. Ranked below history and bookmarks so a user's +/// own visited/saved sites always win; this fills the long tail with +/// well-known destinations (typing "git" -> github.com) even with no history. +class PopularSitesSuggestions extends HookConsumerWidget { + final ValueListenable searchTextListenable; + final void Function(Uri uri) onUriSelected; + + const PopularSitesSuggestions({ + super.key, + required this.searchTextListenable, + required this.onUriSelected, + }); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final results = ref.watch(popularSitesSearchResultsProvider); + + useOnListenableChangeSelector( + searchTextListenable, + () => searchTextListenable.value.text, + () async { + await ref + .read(popularSitesSearchResultsProvider.notifier) + .search(searchTextListenable.value.text); + }, + ); + + if (results.isEmpty) { + return const SliverToBoxAdapter(child: SizedBox.shrink()); + } + + return SearchModuleSection( + title: 'Popular Sites', + moduleType: SearchModuleType.popularSites, + totalCount: results.length, + contentSliverBuilder: + ({required bool isCollapsed, required int visibleCount}) => [ + if (!isCollapsed) + SliverList.builder( + itemCount: visibleCount, + itemBuilder: (context, index) { + final site = results[index]; + final uri = Uri.parse('https://${site.domain}'); + + return ListTile( + key: ValueKey(site.domain), + leading: RepaintBoundary( + child: UrlIcon([uri], iconSize: 24), + ), + title: Text( + site.domain, + maxLines: 1, + overflow: TextOverflow.ellipsis, + ), + onTap: () => onUriSelected(uri), + ); + }, + ), + ], + ); + } +} diff --git a/apps/weblibre/lib/features/popular_sites/data/database/daos/site.dart b/apps/weblibre/lib/features/popular_sites/data/database/daos/site.dart new file mode 100644 index 00000000..77d46c07 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/database/daos/site.dart @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2024-2026 Fabian Freund. + * + * This file is part of WebLibre + * (see https://weblibre.eu). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import 'package:drift/drift.dart'; +import 'package:weblibre/features/popular_sites/data/database/daos/site.drift.dart'; +import 'package:weblibre/features/popular_sites/data/database/database.dart'; +import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart'; + +@DriftAccessor() +class SiteDao extends DatabaseAccessor with $SiteDaoMixin { + SiteDao(super.db); + + /// Popular-domain prefix completions for the omnibar, ordered by Tranco + /// popularity (densely re-ranked at build time, so `rank` is a contiguous + /// 1..N order). [prefix] is matched case-insensitively against the start of + /// the registrable domain. + /// + /// LIKE metacharacters (`%`, `_`, and the `\` escape char itself) in the + /// user input are escaped so they match literally rather than acting as + /// wildcards — see the `ESCAPE '\'` clause on `searchSitesByPrefix`. + Selectable searchByPrefix(String prefix, {int limit = 8}) { + final escaped = prefix + .trim() + .toLowerCase() + .replaceAll(r'\', r'\\') + .replaceAll('%', r'\%') + .replaceAll('_', r'\_'); + + return db.definitionsDrift.searchSitesByPrefix( + pattern: '$escaped%', + limit: limit, + ); + } +} diff --git a/apps/weblibre/lib/features/popular_sites/data/database/daos/site.drift.dart b/apps/weblibre/lib/features/popular_sites/data/database/daos/site.drift.dart new file mode 100644 index 00000000..da53145e --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/database/daos/site.drift.dart @@ -0,0 +1,14 @@ +// dart format width=80 +// ignore_for_file: type=lint +import 'package:drift/drift.dart' as i0; +import 'package:weblibre/features/popular_sites/data/database/database.dart' + as i1; + +mixin $SiteDaoMixin on i0.DatabaseAccessor { + SiteDaoManager get managers => SiteDaoManager(this); +} + +class SiteDaoManager { + final $SiteDaoMixin _db; + SiteDaoManager(this._db); +} diff --git a/apps/weblibre/lib/features/popular_sites/data/database/database.dart b/apps/weblibre/lib/features/popular_sites/data/database/database.dart new file mode 100644 index 00000000..a898c7e6 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/database/database.dart @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024-2026 Fabian Freund. + * + * This file is part of WebLibre + * (see https://weblibre.eu). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import 'package:drift/drift.dart'; +import 'package:drift_dev/api/migrations_native.dart'; +import 'package:flutter/foundation.dart'; +import 'package:weblibre/features/popular_sites/data/database/daos/site.dart'; +import 'package:weblibre/features/popular_sites/data/database/database.drift.dart'; + +@DriftDatabase(include: {'definitions.drift'}, daos: [SiteDao]) +class SitesDatabase extends $SitesDatabase { + @override + final int schemaVersion = 1; + + @override + MigrationStrategy get migration => MigrationStrategy( + beforeOpen: (details) async { + if (kDebugMode) { + await validateDatabaseSchema(); + } + }, + ); + + SitesDatabase(super.e); +} diff --git a/apps/weblibre/lib/features/popular_sites/data/database/database.drift.dart b/apps/weblibre/lib/features/popular_sites/data/database/database.drift.dart new file mode 100644 index 00000000..16c37850 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/database/database.drift.dart @@ -0,0 +1,122 @@ +// dart format width=80 +// ignore_for_file: type=lint +import 'package:drift/drift.dart' as i0; +import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart' + as i1; +import 'package:weblibre/features/popular_sites/data/database/daos/site.dart' + as i2; +import 'package:weblibre/features/popular_sites/data/database/database.dart' + as i3; +import 'package:drift/internal/modular.dart' as i4; +import 'package:sqlite3/common.dart' as i5; + +abstract class $SitesDatabase extends i0.GeneratedDatabase { + $SitesDatabase(i0.QueryExecutor e) : super(e); + $SitesDatabaseManager get managers => $SitesDatabaseManager(this); + late final i1.Sites sites = i1.Sites(this); + late final i2.SiteDao siteDao = i2.SiteDao(this as i3.SitesDatabase); + i1.DefinitionsDrift get definitionsDrift => i4.ReadDatabaseContainer( + this, + ).accessor(i1.DefinitionsDrift.new); + @override + Iterable> get allTables => + allSchemaEntities.whereType>(); + @override + List get allSchemaEntities => [sites]; +} + +class $SitesDatabaseManager { + final $SitesDatabase _db; + $SitesDatabaseManager(this._db); + i1.$SitesTableManager get sites => i1.$SitesTableManager(_db, _db.sites); +} + +extension DefineFunctions on i5.CommonDatabase { + void defineFunctions({ + required String Function(int, String?) lexoRankNext, + required String Function(int, String?) lexoRankPrevious, + required String Function(String?, String?) lexoRankReorderAfter, + required String Function(String?, String?) lexoRankReorderBefore, + required int Function() generateContentHash, + required bool Function(String?) urlIndexable, + required String Function(String?) urlCanonical, + required String Function(String?) urlHost, + required String Function(String?) urlPath, + }) { + createFunction( + functionName: 'lexo_rank_next', + argumentCount: const i5.AllowedArgumentCount(2), + function: (args) { + final arg0 = args[0] as int; + final arg1 = args[1] as String?; + return lexoRankNext(arg0, arg1); + }, + ); + createFunction( + functionName: 'lexo_rank_previous', + argumentCount: const i5.AllowedArgumentCount(2), + function: (args) { + final arg0 = args[0] as int; + final arg1 = args[1] as String?; + return lexoRankPrevious(arg0, arg1); + }, + ); + createFunction( + functionName: 'lexo_rank_reorder_after', + argumentCount: const i5.AllowedArgumentCount(2), + function: (args) { + final arg0 = args[0] as String?; + final arg1 = args[1] as String?; + return lexoRankReorderAfter(arg0, arg1); + }, + ); + createFunction( + functionName: 'lexo_rank_reorder_before', + argumentCount: const i5.AllowedArgumentCount(2), + function: (args) { + final arg0 = args[0] as String?; + final arg1 = args[1] as String?; + return lexoRankReorderBefore(arg0, arg1); + }, + ); + createFunction( + functionName: 'generate_content_hash', + argumentCount: const i5.AllowedArgumentCount(0), + function: (args) { + return generateContentHash(); + }, + ); + createFunction( + functionName: 'url_indexable', + argumentCount: const i5.AllowedArgumentCount(1), + function: (args) { + final arg0 = args[0] as String?; + return urlIndexable(arg0); + }, + ); + createFunction( + functionName: 'url_canonical', + argumentCount: const i5.AllowedArgumentCount(1), + function: (args) { + final arg0 = args[0] as String?; + return urlCanonical(arg0); + }, + ); + createFunction( + functionName: 'url_host', + argumentCount: const i5.AllowedArgumentCount(1), + function: (args) { + final arg0 = args[0] as String?; + return urlHost(arg0); + }, + ); + createFunction( + functionName: 'url_path', + argumentCount: const i5.AllowedArgumentCount(1), + function: (args) { + final arg0 = args[0] as String?; + return urlPath(arg0); + }, + ); + } +} diff --git a/apps/weblibre/lib/features/popular_sites/data/database/definitions.drift b/apps/weblibre/lib/features/popular_sites/data/database/definitions.drift new file mode 100644 index 00000000..e1823b41 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/database/definitions.drift @@ -0,0 +1,13 @@ +CREATE TABLE sites ( + domain TEXT NOT NULL PRIMARY KEY, + rank INTEGER NOT NULL +); + +-- Prefix completion ordered by popularity. The caller escapes LIKE +-- metacharacters in the user input and appends '%', so `ESCAPE '\'` makes any +-- literal % / _ the user typed match literally instead of as wildcards. +searchSitesByPrefix: + SELECT * FROM sites + WHERE domain LIKE :pattern ESCAPE '\' + ORDER BY rank ASC + LIMIT :limit; diff --git a/apps/weblibre/lib/features/popular_sites/data/database/definitions.drift.dart b/apps/weblibre/lib/features/popular_sites/data/database/definitions.drift.dart new file mode 100644 index 00000000..d4d5d4d7 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/database/definitions.drift.dart @@ -0,0 +1,334 @@ +// dart format width=80 +// ignore_for_file: type=lint +import 'package:drift/drift.dart' as i0; +import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart' + as i1; +import 'package:drift/internal/modular.dart' as i2; + +typedef $SitesCreateCompanionBuilder = + i1.SitesCompanion Function({ + required String domain, + required int rank, + i0.Value rowid, + }); +typedef $SitesUpdateCompanionBuilder = + i1.SitesCompanion Function({ + i0.Value domain, + i0.Value rank, + i0.Value rowid, + }); + +class $SitesFilterComposer extends i0.Composer { + $SitesFilterComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + i0.ColumnFilters get domain => $composableBuilder( + column: $table.domain, + builder: (column) => i0.ColumnFilters(column), + ); + + i0.ColumnFilters get rank => $composableBuilder( + column: $table.rank, + builder: (column) => i0.ColumnFilters(column), + ); +} + +class $SitesOrderingComposer + extends i0.Composer { + $SitesOrderingComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + i0.ColumnOrderings get domain => $composableBuilder( + column: $table.domain, + builder: (column) => i0.ColumnOrderings(column), + ); + + i0.ColumnOrderings get rank => $composableBuilder( + column: $table.rank, + builder: (column) => i0.ColumnOrderings(column), + ); +} + +class $SitesAnnotationComposer + extends i0.Composer { + $SitesAnnotationComposer({ + required super.$db, + required super.$table, + super.joinBuilder, + super.$addJoinBuilderToRootComposer, + super.$removeJoinBuilderFromRootComposer, + }); + i0.GeneratedColumn get domain => + $composableBuilder(column: $table.domain, builder: (column) => column); + + i0.GeneratedColumn get rank => + $composableBuilder(column: $table.rank, builder: (column) => column); +} + +class $SitesTableManager + extends + i0.RootTableManager< + i0.GeneratedDatabase, + i1.Sites, + i1.Site, + i1.$SitesFilterComposer, + i1.$SitesOrderingComposer, + i1.$SitesAnnotationComposer, + $SitesCreateCompanionBuilder, + $SitesUpdateCompanionBuilder, + (i1.Site, i0.BaseReferences), + i1.Site, + i0.PrefetchHooks Function() + > { + $SitesTableManager(i0.GeneratedDatabase db, i1.Sites table) + : super( + i0.TableManagerState( + db: db, + table: table, + createFilteringComposer: () => + i1.$SitesFilterComposer($db: db, $table: table), + createOrderingComposer: () => + i1.$SitesOrderingComposer($db: db, $table: table), + createComputedFieldComposer: () => + i1.$SitesAnnotationComposer($db: db, $table: table), + updateCompanionCallback: + ({ + i0.Value domain = const i0.Value.absent(), + i0.Value rank = const i0.Value.absent(), + i0.Value rowid = const i0.Value.absent(), + }) => i1.SitesCompanion(domain: domain, rank: rank, rowid: rowid), + createCompanionCallback: + ({ + required String domain, + required int rank, + i0.Value rowid = const i0.Value.absent(), + }) => i1.SitesCompanion.insert( + domain: domain, + rank: rank, + rowid: rowid, + ), + withReferenceMapper: (p0) => p0 + .map((e) => (e.readTable(table), i0.BaseReferences(db, table, e))) + .toList(), + prefetchHooksCallback: null, + ), + ); +} + +typedef $SitesProcessedTableManager = + i0.ProcessedTableManager< + i0.GeneratedDatabase, + i1.Sites, + i1.Site, + i1.$SitesFilterComposer, + i1.$SitesOrderingComposer, + i1.$SitesAnnotationComposer, + $SitesCreateCompanionBuilder, + $SitesUpdateCompanionBuilder, + (i1.Site, i0.BaseReferences), + i1.Site, + i0.PrefetchHooks Function() + >; + +class Sites extends i0.Table with i0.TableInfo { + @override + final i0.GeneratedDatabase attachedDatabase; + final String? _alias; + Sites(this.attachedDatabase, [this._alias]); + late final i0.GeneratedColumn domain = i0.GeneratedColumn( + 'domain', + aliasedName, + false, + type: i0.DriftSqlType.string, + requiredDuringInsert: true, + $customConstraints: 'NOT NULL PRIMARY KEY', + ); + late final i0.GeneratedColumn rank = i0.GeneratedColumn( + 'rank', + aliasedName, + false, + type: i0.DriftSqlType.int, + requiredDuringInsert: true, + $customConstraints: 'NOT NULL', + ); + @override + List get $columns => [domain, rank]; + @override + String get aliasedName => _alias ?? actualTableName; + @override + String get actualTableName => $name; + static const String $name = 'sites'; + @override + Set get $primaryKey => {domain}; + @override + i1.Site map(Map data, {String? tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : ''; + return i1.Site( + domain: attachedDatabase.typeMapping.read( + i0.DriftSqlType.string, + data['${effectivePrefix}domain'], + )!, + rank: attachedDatabase.typeMapping.read( + i0.DriftSqlType.int, + data['${effectivePrefix}rank'], + )!, + ); + } + + @override + Sites createAlias(String alias) { + return Sites(attachedDatabase, alias); + } + + @override + bool get dontWriteConstraints => true; +} + +class Site extends i0.DataClass implements i0.Insertable { + final String domain; + final int rank; + const Site({required this.domain, required this.rank}); + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + map['domain'] = i0.Variable(domain); + map['rank'] = i0.Variable(rank); + return map; + } + + factory Site.fromJson( + Map json, { + i0.ValueSerializer? serializer, + }) { + serializer ??= i0.driftRuntimeOptions.defaultSerializer; + return Site( + domain: serializer.fromJson(json['domain']), + rank: serializer.fromJson(json['rank']), + ); + } + @override + Map toJson({i0.ValueSerializer? serializer}) { + serializer ??= i0.driftRuntimeOptions.defaultSerializer; + return { + 'domain': serializer.toJson(domain), + 'rank': serializer.toJson(rank), + }; + } + + i1.Site copyWith({String? domain, int? rank}) => + i1.Site(domain: domain ?? this.domain, rank: rank ?? this.rank); + Site copyWithCompanion(i1.SitesCompanion data) { + return Site( + domain: data.domain.present ? data.domain.value : this.domain, + rank: data.rank.present ? data.rank.value : this.rank, + ); + } + + @override + String toString() { + return (StringBuffer('Site(') + ..write('domain: $domain, ') + ..write('rank: $rank') + ..write(')')) + .toString(); + } + + @override + int get hashCode => Object.hash(domain, rank); + @override + bool operator ==(Object other) => + identical(this, other) || + (other is i1.Site && + other.domain == this.domain && + other.rank == this.rank); +} + +class SitesCompanion extends i0.UpdateCompanion { + final i0.Value domain; + final i0.Value rank; + final i0.Value rowid; + const SitesCompanion({ + this.domain = const i0.Value.absent(), + this.rank = const i0.Value.absent(), + this.rowid = const i0.Value.absent(), + }); + SitesCompanion.insert({ + required String domain, + required int rank, + this.rowid = const i0.Value.absent(), + }) : domain = i0.Value(domain), + rank = i0.Value(rank); + static i0.Insertable custom({ + i0.Expression? domain, + i0.Expression? rank, + i0.Expression? rowid, + }) { + return i0.RawValuesInsertable({ + if (domain != null) 'domain': domain, + if (rank != null) 'rank': rank, + if (rowid != null) 'rowid': rowid, + }); + } + + i1.SitesCompanion copyWith({ + i0.Value? domain, + i0.Value? rank, + i0.Value? rowid, + }) { + return i1.SitesCompanion( + domain: domain ?? this.domain, + rank: rank ?? this.rank, + rowid: rowid ?? this.rowid, + ); + } + + @override + Map toColumns(bool nullToAbsent) { + final map = {}; + if (domain.present) { + map['domain'] = i0.Variable(domain.value); + } + if (rank.present) { + map['rank'] = i0.Variable(rank.value); + } + if (rowid.present) { + map['rowid'] = i0.Variable(rowid.value); + } + return map; + } + + @override + String toString() { + return (StringBuffer('SitesCompanion(') + ..write('domain: $domain, ') + ..write('rank: $rank, ') + ..write('rowid: $rowid') + ..write(')')) + .toString(); + } +} + +class DefinitionsDrift extends i2.ModularAccessor { + DefinitionsDrift(i0.GeneratedDatabase db) : super(db); + i0.Selectable searchSitesByPrefix({ + required String pattern, + required int limit, + }) { + return customSelect( + 'SELECT * FROM sites WHERE domain LIKE ?1 ESCAPE \'\\\' ORDER BY rank ASC LIMIT ?2', + variables: [i0.Variable(pattern), i0.Variable(limit)], + readsFrom: {sites}, + ).asyncMap(sites.mapFromRow); + } + + i1.Sites get sites => + i2.ReadDatabaseContainer(attachedDatabase).resultSet('sites'); +} diff --git a/apps/weblibre/lib/features/popular_sites/data/providers.dart b/apps/weblibre/lib/features/popular_sites/data/providers.dart new file mode 100644 index 00000000..71400c54 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/providers.dart @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2024-2026 Fabian Freund. + * + * This file is part of WebLibre + * (see https://weblibre.eu). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import 'dart:io'; + +import 'package:drift/drift.dart'; +import 'package:drift/native.dart'; +import 'package:flutter/services.dart' show rootBundle; +import 'package:path/path.dart' as p; +import 'package:riverpod_annotation/riverpod_annotation.dart'; +import 'package:weblibre/core/database_registry.dart'; +import 'package:weblibre/core/filesystem.dart'; +import 'package:weblibre/features/popular_sites/data/database/database.dart'; + +part 'providers.g.dart'; + +const _sitesAssetPath = 'assets/sites/sites.db'; +const _sitesDbFileName = 'sites.db'; + +@Riverpod(keepAlive: true) +SitesDatabase sitesDatabase(Ref ref) { + final db = SitesDatabase( + LazyDatabase(() async { + final file = File( + p.join(filesystem.profileDatabasesDir.path, _sitesDbFileName), + ); + + await file.parent.create(recursive: true); + + final blob = await rootBundle.load(_sitesAssetPath); + final buffer = blob.buffer; + await file.writeAsBytes( + buffer.asUint8List(blob.offsetInBytes, blob.lengthInBytes), + flush: true, + ); + + return NativeDatabase.createInBackground(file); + }), + ); + + DatabaseRegistry.instance.register('sites', db); + + ref.onDispose(() async { + await db.close(); + }); + + return db; +} diff --git a/apps/weblibre/lib/features/popular_sites/data/providers.g.dart b/apps/weblibre/lib/features/popular_sites/data/providers.g.dart new file mode 100644 index 00000000..17d8f648 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/data/providers.g.dart @@ -0,0 +1,51 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'providers.dart'; + +// ************************************************************************** +// RiverpodGenerator +// ************************************************************************** + +// GENERATED CODE - DO NOT MODIFY BY HAND +// ignore_for_file: type=lint, type=warning + +@ProviderFor(sitesDatabase) +final sitesDatabaseProvider = SitesDatabaseProvider._(); + +final class SitesDatabaseProvider + extends $FunctionalProvider + with $Provider { + SitesDatabaseProvider._() + : super( + from: null, + argument: null, + retry: null, + name: r'sitesDatabaseProvider', + isAutoDispose: false, + dependencies: null, + $allTransitiveDependencies: null, + ); + + @override + String debugGetCreateSourceHash() => _$sitesDatabaseHash(); + + @$internal + @override + $ProviderElement $createElement($ProviderPointer pointer) => + $ProviderElement(pointer); + + @override + SitesDatabase create(Ref ref) { + return sitesDatabase(ref); + } + + /// {@macro riverpod.override_with_value} + Override overrideWithValue(SitesDatabase value) { + return $ProviderOverride( + origin: this, + providerOverride: $SyncValueProvider(value), + ); + } +} + +String _$sitesDatabaseHash() => r'ac11ab9c1da876c18460cd762237698c01b0ec62'; diff --git a/apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.dart b/apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.dart new file mode 100644 index 00000000..a81f8e28 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.dart @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024-2026 Fabian Freund. + * + * This file is part of WebLibre + * (see https://weblibre.eu). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import 'package:riverpod_annotation/riverpod_annotation.dart'; +import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart'; +import 'package:weblibre/features/popular_sites/domain/repositories/popular_sites.dart'; + +part 'popular_sites_search.g.dart'; + +/// Holds the current popular-domain prefix completions for the omnibar +/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers: +/// the widget watches the state list and pushes new queries through [search]. +@Riverpod() +class PopularSitesSearchResults extends _$PopularSitesSearchResults { + Future search(String query, {int limit = 8}) async { + if (query.trim().isEmpty) { + state = const []; + return; + } + + final results = await ref + .read(popularSitesRepositoryProvider.notifier) + .searchByPrefix(query, limit: limit); + + if (!ref.mounted) return; + state = results; + } + + @override + List build() { + return const []; + } +} diff --git a/apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.g.dart b/apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.g.dart new file mode 100644 index 00000000..c114e7b7 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/domain/providers/popular_sites_search.g.dart @@ -0,0 +1,76 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'popular_sites_search.dart'; + +// ************************************************************************** +// RiverpodGenerator +// ************************************************************************** + +// GENERATED CODE - DO NOT MODIFY BY HAND +// ignore_for_file: type=lint, type=warning +/// Holds the current popular-domain prefix completions for the omnibar +/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers: +/// the widget watches the state list and pushes new queries through [search]. + +@ProviderFor(PopularSitesSearchResults) +final popularSitesSearchResultsProvider = PopularSitesSearchResultsProvider._(); + +/// Holds the current popular-domain prefix completions for the omnibar +/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers: +/// the widget watches the state list and pushes new queries through [search]. +final class PopularSitesSearchResultsProvider + extends $NotifierProvider> { + /// Holds the current popular-domain prefix completions for the omnibar + /// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers: + /// the widget watches the state list and pushes new queries through [search]. + PopularSitesSearchResultsProvider._() + : super( + from: null, + argument: null, + retry: null, + name: r'popularSitesSearchResultsProvider', + isAutoDispose: true, + dependencies: null, + $allTransitiveDependencies: null, + ); + + @override + String debugGetCreateSourceHash() => _$popularSitesSearchResultsHash(); + + @$internal + @override + PopularSitesSearchResults create() => PopularSitesSearchResults(); + + /// {@macro riverpod.override_with_value} + Override overrideWithValue(List value) { + return $ProviderOverride( + origin: this, + providerOverride: $SyncValueProvider>(value), + ); + } +} + +String _$popularSitesSearchResultsHash() => + r'107396362aa70acf898b75137df42717ac52f1b3'; + +/// Holds the current popular-domain prefix completions for the omnibar +/// "Popular Sites" module. Mirrors the bookmark/tab search-result notifiers: +/// the widget watches the state list and pushes new queries through [search]. + +abstract class _$PopularSitesSearchResults extends $Notifier> { + List build(); + @$mustCallSuper + @override + WhenComplete runBuild() { + final ref = this.ref as $Ref, List>; + final element = + ref.element + as $ClassProviderElement< + AnyNotifier, List>, + List, + Object?, + Object? + >; + return element.handleCreate(ref, build); + } +} diff --git a/apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.dart b/apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.dart new file mode 100644 index 00000000..6ccf6ed9 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.dart @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2024-2026 Fabian Freund. + * + * This file is part of WebLibre + * (see https://weblibre.eu). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +import 'package:riverpod_annotation/riverpod_annotation.dart'; +import 'package:weblibre/features/popular_sites/data/database/definitions.drift.dart'; +import 'package:weblibre/features/popular_sites/data/providers.dart'; + +part 'popular_sites.g.dart'; + +@Riverpod(keepAlive: true) +class PopularSitesRepository extends _$PopularSitesRepository { + /// Popular registrable domains whose start matches [prefix], ordered by + /// popularity. Returns empty for a blank prefix so the omnibar suggestion + /// module stays quiet until the user types. + Future> searchByPrefix(String prefix, {int limit = 8}) { + if (prefix.trim().isEmpty) { + return Future.value(const []); + } + + return ref + .read(sitesDatabaseProvider) + .siteDao + .searchByPrefix(prefix, limit: limit) + .get(); + } + + @override + void build() {} +} diff --git a/apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.g.dart b/apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.g.dart new file mode 100644 index 00000000..202fbef1 --- /dev/null +++ b/apps/weblibre/lib/features/popular_sites/domain/repositories/popular_sites.g.dart @@ -0,0 +1,63 @@ +// GENERATED CODE - DO NOT MODIFY BY HAND + +part of 'popular_sites.dart'; + +// ************************************************************************** +// RiverpodGenerator +// ************************************************************************** + +// GENERATED CODE - DO NOT MODIFY BY HAND +// ignore_for_file: type=lint, type=warning + +@ProviderFor(PopularSitesRepository) +final popularSitesRepositoryProvider = PopularSitesRepositoryProvider._(); + +final class PopularSitesRepositoryProvider + extends $NotifierProvider { + PopularSitesRepositoryProvider._() + : super( + from: null, + argument: null, + retry: null, + name: r'popularSitesRepositoryProvider', + isAutoDispose: false, + dependencies: null, + $allTransitiveDependencies: null, + ); + + @override + String debugGetCreateSourceHash() => _$popularSitesRepositoryHash(); + + @$internal + @override + PopularSitesRepository create() => PopularSitesRepository(); + + /// {@macro riverpod.override_with_value} + Override overrideWithValue(void value) { + return $ProviderOverride( + origin: this, + providerOverride: $SyncValueProvider(value), + ); + } +} + +String _$popularSitesRepositoryHash() => + r'9825dc634b2ae9ea48a33eac8387a3e138c17b9f'; + +abstract class _$PopularSitesRepository extends $Notifier { + void build(); + @$mustCallSuper + @override + WhenComplete runBuild() { + final ref = this.ref as $Ref; + final element = + ref.element + as $ClassProviderElement< + AnyNotifier, + void, + Object?, + Object? + >; + return element.handleCreate(ref, build); + } +} diff --git a/apps/weblibre/pubspec.yaml b/apps/weblibre/pubspec.yaml index 7013e848..5ac95c86 100644 --- a/apps/weblibre/pubspec.yaml +++ b/apps/weblibre/pubspec.yaml @@ -25,7 +25,7 @@ dependencies: fading_scroll: ^0.9.4 fancy_password_field: ^2.0.8 fast_equatable: ^1.3.1 - file_picker: ^12.0.0-beta.7 + file_picker: 12.0.0-beta.5 flutter: sdk: flutter flutter_auto_size_text: ^5.0.0 @@ -141,6 +141,7 @@ flutter: - assets/preferences/ - assets/legal/ - assets/quotes/ + - assets/sites/ - assets/small_web/ - assets/ublock/ fonts: diff --git a/pubspec.yaml b/pubspec.yaml index 6f682507..8b98703e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -73,12 +73,13 @@ melos: description: Download external assets (bangs, bridges, URL lists) run: bash scripts/update-assets.sh build-components: - description: Build JS components and generate quotes DB + description: Build JS components and generate quotes/sites DBs run: | set -e sh scripts/build-container-proxy.sh sh scripts/build-readability.sh python3 scripts/build_quotes_db.py + python3 scripts/build_sites_db.py build-singbox-libbox: description: Build official sing-box Android libbox AAR for flutter_singbox_proxy run: packages/flutter_singbox_proxy/scripts/build-libbox-android.sh diff --git a/scripts/build_sites_db.py b/scripts/build_sites_db.py new file mode 100644 index 00000000..979b51fa --- /dev/null +++ b/scripts/build_sites_db.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +"""Build a SQLite database of popular websites for omnibar autocomplete. + +Source of truth is the Tranco top-1M ranking (manipulation-resistant aggregate +of Umbrella/Majestic/Cloudflare/Farsight). Tranco ranks *domains by traffic*, +not "sites a human would open", so the head is polluted with adult/gambling +sites, ad/tracker endpoints and pure CDN/infra domains. We filter those out +using: + + * StevenBlack "gambling-porn-only" hosts -> adult + gambling + * Disconnect services.json (Advertising / Analytics / FingerprintingInvasive + / Cryptomining categories only -- NOT Social/Content, which contain + first-party sites people actually visit) -> ad/tracker/fingerprint domains + * a small static denylist of well-known CDN / cloud / API registrable + domains that the tracker lists don't classify + +The survivors (first --limit by rank) are written to a single read-only table +consumed as a bundled asset, mirroring scripts/build_quotes_db.py. + +Raw inputs are produced by `scripts/update-assets.sh --group popular-sites`. +""" + +import argparse +import io +import json +import sqlite3 +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +RAW_DIR = REPO_ROOT / "apps" / "weblibre" / "assets" / "sites" / "raw" +DEFAULT_TRANCO = RAW_DIR / "tranco-top-1m.csv.zip" +DEFAULT_BLOCKLIST = RAW_DIR / "stevenblack-gambling-porn.txt" +DEFAULT_DISCONNECT = RAW_DIR / "disconnect-services.json" +DEFAULT_OUTPUT = REPO_ROOT / "apps" / "weblibre" / "assets" / "sites" / "sites.db" +DEFAULT_TABLE = "sites" + +# How many top-ranked Tranco domains to consider before filtering. Filtering +# removes a sizeable chunk of the head, so we scan well past --limit to still +# fill the quota with genuinely popular sites. +DEFAULT_SCAN_LIMIT = 250_000 +# Final number of clean domains to keep. +DEFAULT_LIMIT = 25_000 + +# Disconnect categories to treat as noise. Social/Content/Email are excluded on +# purpose: they list first-party destinations (facebook.com, twitter.com, ...) +# that users legitimately want autocompleted. +DISCONNECT_BLOCK_CATEGORIES = ( + "Advertising", + "Analytics", + "FingerprintingInvasive", + "Cryptomining", +) + +# Pure CDN / cloud / DNS / registrar / API registrable domains that the tracker +# lists generally don't flag but which are never a navigation target. The +# Disconnect list is great for *trackers* but does not enumerate this infra +# (verified: gtld-servers.net, domaincontrol.com, googletagmanager.com, +# appsflyersdk.com aren't in it at all; googlevideo.com is only under the +# Content category we keep). Kept deliberately famous; the `cdn`-substring rule +# below sweeps the generic-CDN long tail. +STATIC_INFRA_DENYLIST = frozenset( + { + # CDN / cloud edge + "fbcdn.net", + "gstatic.com", + "googleusercontent.com", + "ggpht.com", + "googleapis.com", + "googlevideo.com", + "gvt1.com", + "gvt2.com", + "akamai.net", + "akamaihd.net", + "akamaiedge.net", + "akamaized.net", + "akadns.net", + "edgekey.net", + "edgesuite.net", + "cloudfront.net", + "amazonaws.com", + "azureedge.net", + "windows.net", + "trafficmanager.net", + "fastly.net", + "fastlylb.net", + "llnwd.net", + "stackpathdns.com", + "aaplimg.com", + "apple-dns.net", + # DNS / registry / registrar infra + "gtld-servers.net", + "nstld.com", + "domaincontrol.com", + "ripn.net", + # App SDK / measurement endpoints Disconnect misses + "app-measurement.com", + "googletagmanager.com", + "appsflyersdk.com", + # Device / IoT cloud phone-home (high DNS volume, not navigable) + "ezviz7.com", + "hicloudcam.com", + "whatsapp.net", + } +) + +# Drop any registrable domain containing this token. Verified against the kept +# set: every match is a CDN backend (tiktokcdn, alicdn, spotifycdn, licdn, +# b-cdn, ...) with no first-party site among them, so a plain substring test is +# safe and avoids a broad multi-pattern infra regex. +INFRA_SUBSTRINGS = ("cdn",) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Build a SQLite popular-sites database from a Tranco list." + ) + parser.add_argument("--tranco", default=str(DEFAULT_TRANCO)) + parser.add_argument("--blocklist", default=str(DEFAULT_BLOCKLIST)) + parser.add_argument("--disconnect", default=str(DEFAULT_DISCONNECT)) + parser.add_argument("--output", default=str(DEFAULT_OUTPUT)) + parser.add_argument("--table", default=DEFAULT_TABLE) + parser.add_argument("--scan-limit", type=int, default=DEFAULT_SCAN_LIMIT) + parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT) + return parser.parse_args() + + +def load_tranco(path: Path, scan_limit: int) -> list[tuple[int, str]]: + """Return [(rank, domain)] for the first `scan_limit` Tranco rows. + + Accepts either the raw `rank,domain` CSV or the `.zip` it ships in. + """ + if path.suffix == ".zip": + with zipfile.ZipFile(path) as archive: + csv_name = next(n for n in archive.namelist() if n.endswith(".csv")) + raw = archive.read(csv_name) + handle = io.TextIOWrapper(io.BytesIO(raw), encoding="utf-8") + else: + handle = path.open("r", encoding="utf-8") + + rows: list[tuple[int, str]] = [] + with handle: + for line in handle: + line = line.strip() + if not line: + continue + rank_str, _, domain = line.partition(",") + domain = domain.strip().lower() + if not domain: + continue + try: + rank = int(rank_str) + except ValueError: + continue + rows.append((rank, domain)) + if len(rows) >= scan_limit: + break + return rows + + +def load_hosts_domains(path: Path) -> set[str]: + """Parse a hosts-format file (`0.0.0.0 domain`) into a domain set.""" + domains: set[str] = set() + with path.open("r", encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 2: + continue + domain = parts[1].strip().lower() + if domain and domain not in ("localhost", "localhost.localdomain"): + domains.add(domain) + return domains + + +def load_disconnect_domains(path: Path, categories: tuple[str, ...]) -> set[str]: + """Collect tracker domains from the selected Disconnect categories. + + Structure: categories -> [ {Company: {homepage_url: [domain, ...]}}, ... ]. + Non-list property values (metadata flags) are skipped. + """ + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + + domains: set[str] = set() + all_categories = payload.get("categories", {}) + for category in categories: + for entry in all_categories.get(category, []): + if not isinstance(entry, dict): + continue + for company_props in entry.values(): + if not isinstance(company_props, dict): + continue + for value in company_props.values(): + if isinstance(value, list): + domains.update(d.strip().lower() for d in value if d) + return domains + + +def build_denylist( + blocklist_path: Path, disconnect_path: Path +) -> set[str]: + deny = set(STATIC_INFRA_DENYLIST) + deny |= load_hosts_domains(blocklist_path) + deny |= load_disconnect_domains(disconnect_path, DISCONNECT_BLOCK_CATEGORIES) + return deny + + +def filter_domains( + tranco: list[tuple[int, str]], deny: set[str], limit: int +) -> tuple[list[tuple[str, int]], int]: + """Keep ranked domains not in the denylist, up to `limit`. + + Re-ranks survivors densely (1..N) so the stored rank is a contiguous + popularity order for `ORDER BY rank` queries. Returns `(kept, scanned)` + where `scanned` is how many Tranco rows were consumed to fill the quota, + so the caller can report the real drop rate within the scanned head. + """ + kept: list[tuple[str, int]] = [] + new_rank = 0 + scanned = 0 + for _, domain in tranco: + scanned += 1 + if domain in deny: + continue + if any(token in domain for token in INFRA_SUBSTRINGS): + continue + new_rank += 1 + kept.append((domain, new_rank)) + if len(kept) >= limit: + break + return kept, scanned + + +def quote_identifier(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def build_database( + db_path: Path, table_name: str, rows: list[tuple[str, int]] +) -> None: + db_path.parent.mkdir(parents=True, exist_ok=True) + table = quote_identifier(table_name) + + connection = sqlite3.connect(db_path) + try: + cursor = connection.cursor() + cursor.execute(f"DROP TABLE IF EXISTS {table}") + cursor.execute( + f""" + CREATE TABLE {table} ( + domain TEXT NOT NULL PRIMARY KEY, + rank INTEGER NOT NULL + ) + """ + ) + cursor.executemany( + f"INSERT OR IGNORE INTO {table} (domain, rank) VALUES (?, ?)", + rows, + ) + connection.commit() + cursor.execute("VACUUM") + finally: + connection.close() + + +def main() -> None: + args = parse_args() + + tranco_path = Path(args.tranco).expanduser().resolve() + blocklist_path = Path(args.blocklist).expanduser().resolve() + disconnect_path = Path(args.disconnect).expanduser().resolve() + db_path = Path(args.output).expanduser().resolve() + + # The raw inputs live under assets/sites/raw/ and are gitignored — they are + # produced by `melos run update-assets`. A clean checkout has the committed + # sites.db but not the raw inputs, so `melos run build-components` must not + # fail there: skip rebuilding and keep the committed artifact. Only error if + # there is no committed DB to fall back on. + missing = [p for p in (tranco_path, blocklist_path, disconnect_path) if not p.exists()] + if missing: + names = ", ".join(p.name for p in missing) + if db_path.exists(): + print( + f"Skipping sites.db rebuild: missing raw input(s) [{names}]. " + f"Keeping committed {db_path}. Run " + f"`melos run update-assets` to refresh from source." + ) + return + raise SystemExit( + f"Cannot build {db_path}: missing raw input(s) [{names}] and no " + f"committed DB to fall back on. Run " + f"`melos run update-assets --no-select` first." + ) + + tranco = load_tranco(tranco_path, args.scan_limit) + deny = build_denylist(blocklist_path, disconnect_path) + rows, scanned = filter_domains(tranco, deny, args.limit) + + build_database(db_path, args.table, rows) + + dropped = scanned - len(rows) + drop_pct = (dropped / scanned * 100) if scanned else 0 + print( + f"Denylist={len(deny)} domains. Scanned top {scanned} Tranco ranks to " + f"keep {len(rows)} sites (dropped {dropped}, {drop_pct:.0f}% of head). " + f"Wrote {db_path}." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/update-assets.sh b/scripts/update-assets.sh index b5dd92d4..e0788955 100755 --- a/scripts/update-assets.sh +++ b/scripts/update-assets.sh @@ -104,9 +104,36 @@ update_ublock() { log "uBlock assets sync completed at $(cat "$dir/last_sync.txt")" } +# Raw inputs for the popular-sites database. These are build-time only and are +# consumed by scripts/build_sites_db.py to produce assets/sites/sites.db. The +# raw/ directory is gitignored; only the compiled sites.db is committed. +update_popular_sites() { + local dir="$REPO_ROOT/apps/weblibre/assets/sites/raw" + log "Updating popular-sites source lists..." + + # Tranco top-1M (manipulation-resistant aggregate ranking). The download + # endpoint 307-redirects to the latest list; fetch() follows redirects. + fetch "https://tranco-list.eu/top-1m.csv.zip" \ + "$dir/tranco-top-1m.csv.zip" + + # StevenBlack "gambling-porn-only": the porn + gambling category domains + # WITHOUT the unified malware/ad base list mixed in. + fetch "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts" \ + "$dir/stevenblack-gambling-porn.txt" + + # Disconnect tracker list (basis of Firefox ETP). build_sites_db.py pulls + # only the Advertising/Analytics/FingerprintingInvasive/Cryptomining + # categories so first-party Social/Content sites are preserved. + fetch "https://raw.githubusercontent.com/disconnectme/disconnect-tracking-protection/master/services.json" \ + "$dir/disconnect-services.json" + + date -u --iso-8601=seconds > "$dir/last_sync.txt" + log "Popular-sites sources sync completed at $(cat "$dir/last_sync.txt")" +} + # ── main ───────────────────────────────────────────────────────────────────── -ALL_GROUPS=(bangs bridges url-cleaner url-shorteners ublock) +ALL_GROUPS=(bangs bridges url-cleaner url-shorteners ublock popular-sites) SELECTED_GROUPS=() while [[ $# -gt 0 ]]; do @@ -129,6 +156,7 @@ for group in "${SELECTED_GROUPS[@]}"; do url-cleaner) update_url_cleaner || ((FAILURES++)) ;; url-shorteners) update_url_shorteners || ((FAILURES++)) ;; ublock) update_ublock || ((FAILURES++)) ;; + popular-sites) update_popular_sites || ((FAILURES++)) ;; *) err "Unknown group: $group"; ((FAILURES++)) ;; esac done