From 8214ab166934bd6c664b4700bc34e5f6778f9f56 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 4 Dec 2023 09:36:20 -0700 Subject: [PATCH 01/57] wip --- lib/definitions.dart | 3 ++ lib/sqlite3_common.dart | 2 + lib/src/sqlite_connection.dart | 4 +- lib/src/sqlite_database.dart | 1 + lib/src/sqlite_default_open_factory.dart | 50 ++++++++++++++++++++++++ lib/src/sqlite_open_factory.dart | 6 +-- lib/src/sqlite_queries.dart | 4 +- 7 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 lib/definitions.dart create mode 100644 lib/sqlite3_common.dart create mode 100644 lib/src/sqlite_default_open_factory.dart diff --git a/lib/definitions.dart b/lib/definitions.dart new file mode 100644 index 0000000..4f92f0b --- /dev/null +++ b/lib/definitions.dart @@ -0,0 +1,3 @@ +export 'package:sqlite_async/src/update_notification.dart'; +export 'package:sqlite_async/src/sqlite_connection.dart'; +export 'package:sqlite_async/src/sqlite_queries.dart'; diff --git a/lib/sqlite3_common.dart b/lib/sqlite3_common.dart new file mode 100644 index 0000000..4c1ddc0 --- /dev/null +++ b/lib/sqlite3_common.dart @@ -0,0 +1,2 @@ +// Exports common Sqlite3 exports which are available on web and ffi environments +export 'package:sqlite3/common.dart'; diff --git a/lib/src/sqlite_connection.dart b/lib/src/sqlite_connection.dart index fb027f3..6df5ba8 100644 --- a/lib/src/sqlite_connection.dart +++ b/lib/src/sqlite_connection.dart @@ -1,4 +1,4 @@ -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite3/common.dart' as sqlite; /// Abstract class representing calls available in a read-only or read-write context. abstract class SqliteReadContext { @@ -43,7 +43,7 @@ abstract class SqliteReadContext { /// } /// ``` Future computeWithDatabase( - Future Function(sqlite.Database db) compute); + Future Function(sqlite.CommonDatabase db) compute); } /// Abstract class representing calls available in a read-write context. diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 01588c8..39f4ea3 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -9,6 +9,7 @@ import 'port_channel.dart'; import 'sqlite_connection.dart'; import 'sqlite_connection_impl.dart'; import 'sqlite_open_factory.dart'; +import 'sqlite_default_open_factory.dart'; import 'sqlite_options.dart'; import 'sqlite_queries.dart'; import 'update_notification.dart'; diff --git a/lib/src/sqlite_default_open_factory.dart b/lib/src/sqlite_default_open_factory.dart new file mode 100644 index 0000000..9fe875d --- /dev/null +++ b/lib/src/sqlite_default_open_factory.dart @@ -0,0 +1,50 @@ +import './sqlite_open_factory.dart'; +import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite3/common.dart' as sqlite_common; +import 'sqlite_options.dart'; + +/// The default database factory. +/// +/// This takes care of opening the database, and running PRAGMA statements +/// to configure the connection. +/// +/// Override the [open] method to customize the process. +class DefaultSqliteOpenFactory implements SqliteOpenFactory { + final String path; + final SqliteOptions sqliteOptions; + + const DefaultSqliteOpenFactory( + {required this.path, + this.sqliteOptions = const SqliteOptions.defaults()}); + + List pragmaStatements(SqliteOpenOptions options) { + List statements = []; + + if (options.primaryConnection && sqliteOptions.journalMode != null) { + // Persisted - only needed on the primary connection + statements + .add('PRAGMA journal_mode = ${sqliteOptions.journalMode!.name}'); + } + if (!options.readOnly && sqliteOptions.journalSizeLimit != null) { + // Needed on every writable connection + statements.add( + 'PRAGMA journal_size_limit = ${sqliteOptions.journalSizeLimit!}'); + } + if (sqliteOptions.synchronous != null) { + // Needed on every connection + statements.add('PRAGMA synchronous = ${sqliteOptions.synchronous!.name}'); + } + return statements; + } + + @override + sqlite_common.CommonDatabase open(SqliteOpenOptions options) { + final mode = options.openMode; + var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); + + for (var statement in pragmaStatements(options)) { + db.execute(statement); + } + return db; + } +} diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 4f1845a..3cc23a3 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,6 +1,6 @@ import 'dart:async'; -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite3/common.dart' as sqlite; import 'sqlite_options.dart'; @@ -9,7 +9,7 @@ import 'sqlite_options.dart'; /// Since connections are opened in dedicated background isolates, this class /// must be safe to pass to different isolates. abstract class SqliteOpenFactory { - FutureOr open(SqliteOpenOptions options); + FutureOr open(SqliteOpenOptions options); } /// The default database factory. @@ -47,7 +47,7 @@ class DefaultSqliteOpenFactory implements SqliteOpenFactory { } @override - sqlite.Database open(SqliteOpenOptions options) { + sqlite.CommonDatabase open(SqliteOpenOptions options) { final mode = options.openMode; var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); diff --git a/lib/src/sqlite_queries.dart b/lib/src/sqlite_queries.dart index 055c11c..1b53e3c 100644 --- a/lib/src/sqlite_queries.dart +++ b/lib/src/sqlite_queries.dart @@ -1,4 +1,4 @@ -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite3/common.dart' as sqlite; import 'database_utils.dart'; import 'sqlite_connection.dart'; @@ -122,7 +122,7 @@ mixin SqliteQueries implements SqliteWriteContext, SqliteConnection { /// write transaction. @override Future computeWithDatabase( - Future Function(sqlite.Database db) compute) { + Future Function(sqlite.CommonDatabase db) compute) { return writeTransaction((tx) async { return tx.computeWithDatabase(compute); }); From 7f7a05ed73822a01e753c88f9604cdf9509b9b8f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 6 Dec 2023 15:55:29 -0700 Subject: [PATCH 02/57] common database types --- lib/definitions.dart | 2 + lib/sqlite_async.dart | 3 +- lib/src/default_sqlite_open_factory.dart | 21 ++++++++ lib/src/isolate_connection_factory.dart | 2 +- lib/src/sqlite_connection_impl.dart | 6 +-- lib/src/sqlite_database.dart | 5 +- lib/src/sqlite_default_open_factory.dart | 50 ------------------- lib/src/sqlite_open_factory.dart | 62 ++++++++++-------------- test/util.dart | 2 +- 9 files changed, 59 insertions(+), 94 deletions(-) create mode 100644 lib/src/default_sqlite_open_factory.dart delete mode 100644 lib/src/sqlite_default_open_factory.dart diff --git a/lib/definitions.dart b/lib/definitions.dart index 4f92f0b..5f486db 100644 --- a/lib/definitions.dart +++ b/lib/definitions.dart @@ -1,3 +1,5 @@ export 'package:sqlite_async/src/update_notification.dart'; export 'package:sqlite_async/src/sqlite_connection.dart'; export 'package:sqlite_async/src/sqlite_queries.dart'; +export 'package:sqlite_async/src/sqlite_open_factory.dart'; +export 'src/sqlite_options.dart'; diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index 1d6e5dd..e69829f 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -3,12 +3,13 @@ /// See [SqliteDatabase] as a starting point. library; +export 'src/default_sqlite_open_factory.dart'; export 'src/isolate_connection_factory.dart'; export 'src/sqlite_connection.dart'; export 'src/sqlite_database.dart'; export 'src/sqlite_migrations.dart'; export 'src/sqlite_open_factory.dart'; -export 'src/sqlite_options.dart'; export 'src/sqlite_queries.dart'; export 'src/update_notification.dart'; export 'src/utils.dart'; +export 'definitions.dart'; diff --git a/lib/src/default_sqlite_open_factory.dart b/lib/src/default_sqlite_open_factory.dart new file mode 100644 index 0000000..2d4f0cf --- /dev/null +++ b/lib/src/default_sqlite_open_factory.dart @@ -0,0 +1,21 @@ +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_async/src/sqlite_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; + +class DefaultSqliteOpenFactory + extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}); + + @override + Database open(SqliteOpenOptions options) { + final mode = options.openMode; + var db = sqlite3.open(path, mode: mode, mutex: false); + + for (var statement in pragmaStatements(options)) { + db.execute(statement); + } + return db; + } +} diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/isolate_connection_factory.dart index d6ab74d..27c434d 100644 --- a/lib/src/isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory.dart @@ -13,7 +13,7 @@ import 'update_notification.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory { - SqliteOpenFactory openFactory; + SqliteOpenFactory openFactory; SerializedMutex mutex; SerializedPortClient upstreamPort; diff --git a/lib/src/sqlite_connection_impl.dart b/lib/src/sqlite_connection_impl.dart index e0a6ff6..be281a9 100644 --- a/lib/src/sqlite_connection_impl.dart +++ b/lib/src/sqlite_connection_impl.dart @@ -29,7 +29,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { final bool readOnly; SqliteConnectionImpl( - {required SqliteOpenFactory openFactory, + {required SqliteOpenFactory openFactory, required Mutex mutex, required SerializedPortClient upstreamPort, this.updates, @@ -49,7 +49,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { return _isolateClient.closed; } - Future _open(SqliteOpenFactory openFactory, + Future _open(SqliteOpenFactory openFactory, {required bool primary, required SerializedPortClient upstreamPort}) async { await _connectionMutex.lock(() async { @@ -334,7 +334,7 @@ class _SqliteConnectionParams { final SerializedPortClient port; final bool primary; - final SqliteOpenFactory openFactory; + final SqliteOpenFactory openFactory; _SqliteConnectionParams( {required this.openFactory, diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 39f4ea3..c212383 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -3,16 +3,17 @@ import 'dart:isolate'; import 'connection_pool.dart'; import 'database_utils.dart'; +import 'default_sqlite_open_factory.dart'; import 'isolate_connection_factory.dart'; import 'mutex.dart'; import 'port_channel.dart'; import 'sqlite_connection.dart'; import 'sqlite_connection_impl.dart'; import 'sqlite_open_factory.dart'; -import 'sqlite_default_open_factory.dart'; import 'sqlite_options.dart'; import 'sqlite_queries.dart'; import 'update_notification.dart'; +import 'package:sqlite3/sqlite3.dart'; /// A SQLite database instance. /// @@ -33,7 +34,7 @@ class SqliteDatabase with SqliteQueries implements SqliteConnection { /// This must be safe to pass to different isolates. /// /// Use a custom class for this to customize the open process. - final SqliteOpenFactory openFactory; + final AbstractDefaultSqliteOpenFactory openFactory; /// Use this stream to subscribe to notifications of updates to tables. @override diff --git a/lib/src/sqlite_default_open_factory.dart b/lib/src/sqlite_default_open_factory.dart deleted file mode 100644 index 9fe875d..0000000 --- a/lib/src/sqlite_default_open_factory.dart +++ /dev/null @@ -1,50 +0,0 @@ -import './sqlite_open_factory.dart'; -import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite3/common.dart' as sqlite_common; -import 'sqlite_options.dart'; - -/// The default database factory. -/// -/// This takes care of opening the database, and running PRAGMA statements -/// to configure the connection. -/// -/// Override the [open] method to customize the process. -class DefaultSqliteOpenFactory implements SqliteOpenFactory { - final String path; - final SqliteOptions sqliteOptions; - - const DefaultSqliteOpenFactory( - {required this.path, - this.sqliteOptions = const SqliteOptions.defaults()}); - - List pragmaStatements(SqliteOpenOptions options) { - List statements = []; - - if (options.primaryConnection && sqliteOptions.journalMode != null) { - // Persisted - only needed on the primary connection - statements - .add('PRAGMA journal_mode = ${sqliteOptions.journalMode!.name}'); - } - if (!options.readOnly && sqliteOptions.journalSizeLimit != null) { - // Needed on every writable connection - statements.add( - 'PRAGMA journal_size_limit = ${sqliteOptions.journalSizeLimit!}'); - } - if (sqliteOptions.synchronous != null) { - // Needed on every connection - statements.add('PRAGMA synchronous = ${sqliteOptions.synchronous!.name}'); - } - return statements; - } - - @override - sqlite_common.CommonDatabase open(SqliteOpenOptions options) { - final mode = options.openMode; - var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); - - for (var statement in pragmaStatements(options)) { - db.execute(statement); - } - return db; - } -} diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 3cc23a3..812e822 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -8,8 +8,29 @@ import 'sqlite_options.dart'; /// /// Since connections are opened in dedicated background isolates, this class /// must be safe to pass to different isolates. -abstract class SqliteOpenFactory { - FutureOr open(SqliteOpenOptions options); +abstract class SqliteOpenFactory { + FutureOr open(SqliteOpenOptions options); +} + +class SqliteOpenOptions { + /// Whether this is the primary write connection for the database. + final bool primaryConnection; + + /// Whether this connection is read-only. + final bool readOnly; + + const SqliteOpenOptions( + {required this.primaryConnection, required this.readOnly}); + + sqlite.OpenMode get openMode { + if (primaryConnection) { + return sqlite.OpenMode.readWriteCreate; + } else if (readOnly) { + return sqlite.OpenMode.readOnly; + } else { + return sqlite.OpenMode.readWrite; + } + } } /// The default database factory. @@ -18,11 +39,12 @@ abstract class SqliteOpenFactory { /// to configure the connection. /// /// Override the [open] method to customize the process. -class DefaultSqliteOpenFactory implements SqliteOpenFactory { +abstract class AbstractDefaultSqliteOpenFactory + implements SqliteOpenFactory { final String path; final SqliteOptions sqliteOptions; - const DefaultSqliteOpenFactory( + const AbstractDefaultSqliteOpenFactory( {required this.path, this.sqliteOptions = const SqliteOptions.defaults()}); @@ -45,36 +67,4 @@ class DefaultSqliteOpenFactory implements SqliteOpenFactory { } return statements; } - - @override - sqlite.CommonDatabase open(SqliteOpenOptions options) { - final mode = options.openMode; - var db = sqlite.sqlite3.open(path, mode: mode, mutex: false); - - for (var statement in pragmaStatements(options)) { - db.execute(statement); - } - return db; - } -} - -class SqliteOpenOptions { - /// Whether this is the primary write connection for the database. - final bool primaryConnection; - - /// Whether this connection is read-only. - final bool readOnly; - - const SqliteOpenOptions( - {required this.primaryConnection, required this.readOnly}); - - sqlite.OpenMode get openMode { - if (primaryConnection) { - return sqlite.OpenMode.readWriteCreate; - } else if (readOnly) { - return sqlite.OpenMode.readOnly; - } else { - return sqlite.OpenMode.readWrite; - } - } } diff --git a/test/util.dart b/test/util.dart index 12458fc..55e8367 100644 --- a/test/util.dart +++ b/test/util.dart @@ -49,7 +49,7 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { } } -SqliteOpenFactory testFactory({String? path}) { +SqliteOpenFactory testFactory({String? path}) { return TestSqliteOpenFactory(path: path ?? dbPath()); } From 9658a6bc6ab67e0e710ec591d8cd0649c7504d2f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 6 Dec 2023 16:09:51 -0700 Subject: [PATCH 03/57] added abstractions --- lib/src/connection_pool.dart | 4 +++- lib/src/default_sqlite_open_factory.dart | 2 ++ lib/src/sqlite_database.dart | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/src/connection_pool.dart b/lib/src/connection_pool.dart index 62192a3..16786dc 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/connection_pool.dart @@ -1,5 +1,7 @@ import 'dart:async'; +import 'package:sqlite3/sqlite3.dart'; + import 'mutex.dart'; import 'port_channel.dart'; import 'sqlite_connection.dart'; @@ -14,7 +16,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final List _readConnections = []; - final SqliteOpenFactory _factory; + final SqliteOpenFactory _factory; final SerializedPortClient _upstreamPort; @override diff --git a/lib/src/default_sqlite_open_factory.dart b/lib/src/default_sqlite_open_factory.dart index 2d4f0cf..35fe532 100644 --- a/lib/src/default_sqlite_open_factory.dart +++ b/lib/src/default_sqlite_open_factory.dart @@ -11,6 +11,8 @@ class DefaultSqliteOpenFactory @override Database open(SqliteOpenOptions options) { final mode = options.openMode; + + print('ttt' + path + mode.toString()); var db = sqlite3.open(path, mode: mode, mutex: false); for (var statement in pragmaStatements(options)) { diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index c212383..8738aa1 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -34,7 +34,7 @@ class SqliteDatabase with SqliteQueries implements SqliteConnection { /// This must be safe to pass to different isolates. /// /// Use a custom class for this to customize the open process. - final AbstractDefaultSqliteOpenFactory openFactory; + final SqliteOpenFactory openFactory; /// Use this stream to subscribe to notifications of updates to tables. @override From 13c6d94f306073b0f7ca207b0abb0fa76a87aa2f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 8 Jan 2024 15:30:52 +0200 Subject: [PATCH 04/57] remove test log --- lib/src/default_sqlite_open_factory.dart | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/src/default_sqlite_open_factory.dart b/lib/src/default_sqlite_open_factory.dart index 35fe532..a8b05ef 100644 --- a/lib/src/default_sqlite_open_factory.dart +++ b/lib/src/default_sqlite_open_factory.dart @@ -12,7 +12,6 @@ class DefaultSqliteOpenFactory Database open(SqliteOpenOptions options) { final mode = options.openMode; - print('ttt' + path + mode.toString()); var db = sqlite3.open(path, mode: mode, mutex: false); for (var statement in pragmaStatements(options)) { From 8ae36c8ed0f2282c695a72396ec0c8174c531389 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 10 Jan 2024 13:26:57 +0200 Subject: [PATCH 05/57] added web abstractions --- example/custom_functions_example.dart | 8 +- example/linux_cli_example.dart | 4 +- lib/definitions.dart | 2 +- lib/sqlite_async.dart | 2 - .../database/abstract_sqlite_database.dart | 79 ++++++ .../native}/connection_pool.dart | 10 +- .../native}/isolate_connection_factory.dart | 10 +- .../native/native_sqlite_database.dart | 222 ++++++++++++++++ .../{ => database/native}/port_channel.dart | 0 .../native}/sqlite_connection_impl.dart | 12 +- lib/src/database/stub_sqlite_database.dart | 39 +++ lib/src/database/web/web_sqlite_database.dart | 26 ++ lib/src/default_sqlite_open_factory.dart | 22 -- lib/src/mutex.dart | 2 +- .../open_factory/abstract_open_factory.dart | 63 +++++ .../native/native_sqlite_open_factory.dart | 38 +++ .../stub_sqlite_open_factory.dart | 19 ++ .../web/web_sqlite_open_factory.dart | 25 ++ lib/src/sqlite_database.dart | 239 +----------------- lib/src/sqlite_migrations.dart | 2 +- lib/src/sqlite_open_factory.dart | 75 +----- lib/src/sqlite_options.dart | 19 +- lib/src/sqlite_queries.dart | 2 +- lib/src/utils/database_utils.dart | 2 + lib/src/utils/native_database_utils.dart | 13 + .../shared_utils.dart} | 15 +- lib/utils.dart | 1 + scripts/benchmark.dart | 28 +- test/basic_test.dart | 3 +- test/close_test.dart | 3 +- test/json1_test.dart | 3 +- test/util.dart | 5 +- test/watch_test.dart | 6 +- 33 files changed, 614 insertions(+), 385 deletions(-) create mode 100644 lib/src/database/abstract_sqlite_database.dart rename lib/src/{ => database/native}/connection_pool.dart (96%) rename lib/src/{ => database/native}/isolate_connection_factory.dart (93%) create mode 100644 lib/src/database/native/native_sqlite_database.dart rename lib/src/{ => database/native}/port_channel.dart (100%) rename lib/src/{ => database/native}/sqlite_connection_impl.dart (97%) create mode 100644 lib/src/database/stub_sqlite_database.dart create mode 100644 lib/src/database/web/web_sqlite_database.dart delete mode 100644 lib/src/default_sqlite_open_factory.dart create mode 100644 lib/src/open_factory/abstract_open_factory.dart create mode 100644 lib/src/open_factory/native/native_sqlite_open_factory.dart create mode 100644 lib/src/open_factory/stub_sqlite_open_factory.dart create mode 100644 lib/src/open_factory/web/web_sqlite_open_factory.dart create mode 100644 lib/src/utils/database_utils.dart create mode 100644 lib/src/utils/native_database_utils.dart rename lib/src/{database_utils.dart => utils/shared_utils.dart} (91%) create mode 100644 lib/utils.dart diff --git a/example/custom_functions_example.dart b/example/custom_functions_example.dart index bede8d6..c2e9c9d 100644 --- a/example/custom_functions_example.dart +++ b/example/custom_functions_example.dart @@ -1,8 +1,8 @@ import 'dart:io'; import 'dart:isolate'; +import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite3/sqlite3.dart' as sqlite; /// Since the functions need to be created on every SQLite connection, /// we do this in a SqliteOpenFactory. @@ -10,12 +10,12 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { TestOpenFactory({required super.path, super.sqliteOptions}); @override - sqlite.Database open(SqliteOpenOptions options) { + CommonDatabase open(SqliteOpenOptions options) { final db = super.open(options); db.createFunction( functionName: 'sleep', - argumentCount: const sqlite.AllowedArgumentCount(1), + argumentCount: const AllowedArgumentCount(1), function: (args) { final millis = args[0] as int; sleep(Duration(milliseconds: millis)); @@ -25,7 +25,7 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { db.createFunction( functionName: 'isolate_name', - argumentCount: const sqlite.AllowedArgumentCount(0), + argumentCount: const AllowedArgumentCount(0), function: (args) { return Isolate.current.debugName; }, diff --git a/example/linux_cli_example.dart b/example/linux_cli_example.dart index 82cfee9..4a3462c 100644 --- a/example/linux_cli_example.dart +++ b/example/linux_cli_example.dart @@ -1,8 +1,8 @@ import 'dart:ffi'; +import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite3/open.dart' as sqlite_open; -import 'package:sqlite3/sqlite3.dart' as sqlite; const defaultSqlitePath = 'libsqlite3.so.0'; @@ -16,7 +16,7 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { this.sqlitePath = defaultSqlitePath}); @override - sqlite.Database open(SqliteOpenOptions options) { + CommonDatabase open(SqliteOpenOptions options) { // For details, see: // https://pub.dev/packages/sqlite3#manually-providing-sqlite3-libraries sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { diff --git a/lib/definitions.dart b/lib/definitions.dart index 5f486db..96ecb21 100644 --- a/lib/definitions.dart +++ b/lib/definitions.dart @@ -2,4 +2,4 @@ export 'package:sqlite_async/src/update_notification.dart'; export 'package:sqlite_async/src/sqlite_connection.dart'; export 'package:sqlite_async/src/sqlite_queries.dart'; export 'package:sqlite_async/src/sqlite_open_factory.dart'; -export 'src/sqlite_options.dart'; +export 'package:sqlite_async/src/sqlite_options.dart'; diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index e69829f..5e51c89 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -3,8 +3,6 @@ /// See [SqliteDatabase] as a starting point. library; -export 'src/default_sqlite_open_factory.dart'; -export 'src/isolate_connection_factory.dart'; export 'src/sqlite_connection.dart'; export 'src/sqlite_database.dart'; export 'src/sqlite_migrations.dart'; diff --git a/lib/src/database/abstract_sqlite_database.dart b/lib/src/database/abstract_sqlite_database.dart new file mode 100644 index 0000000..26d59ff --- /dev/null +++ b/lib/src/database/abstract_sqlite_database.dart @@ -0,0 +1,79 @@ +import 'dart:async'; + +import '../../definitions.dart'; +import '../sqlite_connection.dart'; +import '../sqlite_queries.dart'; +import '../update_notification.dart'; + +/// A SQLite database instance. +/// +/// Use one instance per database file. If multiple instances are used, update +/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. +abstract class AbstractSqliteDatabase + with SqliteQueries + implements SqliteConnection { + /// The maximum number of concurrent read transactions if not explicitly specified. + static const int defaultMaxReaders = 5; + + /// Maximum number of concurrent read transactions. + late final int maxReaders; + + /// Factory that opens a raw database connection in each isolate. + /// + /// This must be safe to pass to different isolates. + /// + /// Use a custom class for this to customize the open process. + late final SqliteOpenFactory openFactory; + + /// Use this stream to subscribe to notifications of updates to tables. + @override + late final Stream updates; + + final StreamController _updatesController = + StreamController.broadcast(); + + late final Future _initialized; + + Future _init(); + + /// Wait for initialization to complete. + /// + /// While initializing is automatic, this helps to catch and report initialization errors. + Future initialize() async { + await _initialized; + } + + /// Open a read-only transaction. + /// + /// Up to [maxReaders] read transactions can run concurrently. + /// After that, read transactions are queued. + /// + /// Read transactions can run concurrently to a write transaction. + /// + /// Changes from any write transaction are not visible to read transactions + /// started before it. + @override + Future readTransaction( + Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout}); + + /// Open a read-write transaction. + /// + /// Only a single write transaction can run at a time - any concurrent + /// transactions are queued. + /// + /// The write transaction is automatically committed when the callback finishes, + /// or rolled back on any error. + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout}); + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}); + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}); +} diff --git a/lib/src/connection_pool.dart b/lib/src/database/native/connection_pool.dart similarity index 96% rename from lib/src/connection_pool.dart rename to lib/src/database/native/connection_pool.dart index 16786dc..80252ab 100644 --- a/lib/src/connection_pool.dart +++ b/lib/src/database/native/connection_pool.dart @@ -2,13 +2,13 @@ import 'dart:async'; import 'package:sqlite3/sqlite3.dart'; -import 'mutex.dart'; +import '../../mutex.dart'; +import '../../sqlite_connection.dart'; +import '../../sqlite_open_factory.dart'; +import '../../sqlite_queries.dart'; +import '../../update_notification.dart'; import 'port_channel.dart'; -import 'sqlite_connection.dart'; import 'sqlite_connection_impl.dart'; -import 'sqlite_open_factory.dart'; -import 'sqlite_queries.dart'; -import 'update_notification.dart'; /// A connection pool with a single write connection and multiple read connections. class SqliteConnectionPool with SqliteQueries implements SqliteConnection { diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/database/native/isolate_connection_factory.dart similarity index 93% rename from lib/src/isolate_connection_factory.dart rename to lib/src/database/native/isolate_connection_factory.dart index 27c434d..34ac8c2 100644 --- a/lib/src/isolate_connection_factory.dart +++ b/lib/src/database/native/isolate_connection_factory.dart @@ -3,13 +3,13 @@ import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'database_utils.dart'; -import 'mutex.dart'; +import '../../mutex.dart'; +import '../../sqlite_connection.dart'; +import '../../sqlite_open_factory.dart'; +import '../../update_notification.dart'; +import '../../utils/native_database_utils.dart'; import 'port_channel.dart'; -import 'sqlite_connection.dart'; import 'sqlite_connection_impl.dart'; -import 'sqlite_open_factory.dart'; -import 'update_notification.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory { diff --git a/lib/src/database/native/native_sqlite_database.dart b/lib/src/database/native/native_sqlite_database.dart new file mode 100644 index 0000000..646bc84 --- /dev/null +++ b/lib/src/database/native/native_sqlite_database.dart @@ -0,0 +1,222 @@ +import 'dart:async'; +import 'dart:isolate'; + +import 'package:sqlite3/sqlite3.dart'; + +import '../../../mutex.dart'; +import '../../utils/database_utils.dart'; +import '../../sqlite_connection.dart'; +import '../../open_factory/native/native_sqlite_open_factory.dart'; +import '../../open_factory/abstract_open_factory.dart'; +import '../../sqlite_options.dart'; +import '../../update_notification.dart'; +import '../abstract_sqlite_database.dart'; +import 'port_channel.dart'; +import 'connection_pool.dart'; +import 'sqlite_connection_impl.dart'; +import 'isolate_connection_factory.dart'; + +/// A SQLite database instance. +/// +/// Use one instance per database file. If multiple instances are used, update +/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. +class SqliteDatabase extends AbstractSqliteDatabase { + /// Use this stream to subscribe to notifications of updates to tables. + @override + late final Stream updates; + + final SqliteOpenFactory openFactory; + + final StreamController _updatesController = + StreamController.broadcast(); + + late final PortServer _eventsPort; + + late final SqliteConnectionImpl _internalConnection; + late final SqliteConnectionPool _pool; + late final Future _initialized; + + /// Global lock to serialize write transactions. + final SimpleMutex mutex = SimpleMutex(); + + /// Open a SqliteDatabase. + /// + /// Only a single SqliteDatabase per [path] should be opened at a time. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + factory SqliteDatabase( + {required path, + int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + final factory = + DefaultSqliteOpenFactory(path: path, sqliteOptions: options); + return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + } + + /// Advanced: Open a database with a specified factory. + /// + /// The factory is used to open each database connection in background isolates. + /// + /// Use when control is required over the opening process. Examples include: + /// 1. Specifying the path to `libsqlite.so` on Linux. + /// 2. Running additional per-connection PRAGMA statements on each connection. + /// 3. Creating custom SQLite functions. + /// 4. Creating temporary views or triggers. + SqliteDatabase.withFactory(this.openFactory, + {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + updates = _updatesController.stream; + + super.maxReaders = maxReaders; + + _listenForEvents(); + + _internalConnection = _openPrimaryConnection(debugName: 'sqlite-writer'); + _pool = SqliteConnectionPool(openFactory, + upstreamPort: _eventsPort.client(), + updates: updates, + writeConnection: _internalConnection, + debugName: 'sqlite', + maxReaders: maxReaders, + mutex: mutex); + + _initialized = _init(); + } + + Future _init() async { + await _internalConnection.ready; + } + + /// Wait for initialization to complete. + /// + /// While initializing is automatic, this helps to catch and report initialization errors. + Future initialize() async { + await _initialized; + } + + @override + bool get closed { + return _pool.closed; + } + + void _listenForEvents() { + UpdateNotification? updates; + + Map subscriptions = {}; + + _eventsPort = PortServer((message) async { + if (message is UpdateNotification) { + if (updates == null) { + updates = message; + // Use the mutex to only send updates after the current transaction. + // Do take care to avoid getting a lock for each individual update - + // that could add massive performance overhead. + mutex.lock(() async { + if (updates != null) { + _updatesController.add(updates!); + updates = null; + } + }); + } else { + updates!.tables.addAll(message.tables); + } + return null; + } else if (message is InitDb) { + await _initialized; + return null; + } else if (message is SubscribeToUpdates) { + if (subscriptions.containsKey(message.port)) { + return; + } + final subscription = _updatesController.stream.listen((event) { + message.port.send(event); + }); + subscriptions[message.port] = subscription; + return null; + } else if (message is UnsubscribeToUpdates) { + final subscription = subscriptions.remove(message.port); + subscription?.cancel(); + return null; + } else { + throw ArgumentError('Unknown message type: $message'); + } + }); + } + + /// A connection factory that can be passed to different isolates. + /// + /// Use this to access the database in background isolates. + IsolateConnectionFactory isolateConnectionFactory() { + return IsolateConnectionFactory( + openFactory: openFactory, + mutex: mutex.shared, + upstreamPort: _eventsPort.client()); + } + + SqliteConnectionImpl _openPrimaryConnection({String? debugName}) { + return SqliteConnectionImpl( + upstreamPort: _eventsPort.client(), + primary: true, + updates: updates, + debugName: debugName, + mutex: mutex, + readOnly: false, + openFactory: openFactory); + } + + @override + Future close() async { + await _pool.close(); + _updatesController.close(); + _eventsPort.close(); + await mutex.close(); + } + + /// Open a read-only transaction. + /// + /// Up to [maxReaders] read transactions can run concurrently. + /// After that, read transactions are queued. + /// + /// Read transactions can run concurrently to a write transaction. + /// + /// Changes from any write transaction are not visible to read transactions + /// started before it. + @override + Future readTransaction( + Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout}) { + return _pool.readTransaction(callback, lockTimeout: lockTimeout); + } + + /// Open a read-write transaction. + /// + /// Only a single write transaction can run at a time - any concurrent + /// transactions are queued. + /// + /// The write transaction is automatically committed when the callback finishes, + /// or rolled back on any error. + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout}) { + return _pool.writeTransaction(callback, lockTimeout: lockTimeout); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return _pool.readLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return _pool.writeLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } +} diff --git a/lib/src/port_channel.dart b/lib/src/database/native/port_channel.dart similarity index 100% rename from lib/src/port_channel.dart rename to lib/src/database/native/port_channel.dart diff --git a/lib/src/sqlite_connection_impl.dart b/lib/src/database/native/sqlite_connection_impl.dart similarity index 97% rename from lib/src/sqlite_connection_impl.dart rename to lib/src/database/native/sqlite_connection_impl.dart index be281a9..b93cd6b 100644 --- a/lib/src/sqlite_connection_impl.dart +++ b/lib/src/database/native/sqlite_connection_impl.dart @@ -3,13 +3,13 @@ import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'database_utils.dart'; -import 'mutex.dart'; +import '../../utils/database_utils.dart'; +import '../../mutex.dart'; import 'port_channel.dart'; -import 'sqlite_connection.dart'; -import 'sqlite_open_factory.dart'; -import 'sqlite_queries.dart'; -import 'update_notification.dart'; +import '../../sqlite_connection.dart'; +import '../../sqlite_open_factory.dart'; +import '../../sqlite_queries.dart'; +import '../../update_notification.dart'; typedef TxCallback = Future Function(sqlite.Database db); diff --git a/lib/src/database/stub_sqlite_database.dart b/lib/src/database/stub_sqlite_database.dart new file mode 100644 index 0000000..c58ee12 --- /dev/null +++ b/lib/src/database/stub_sqlite_database.dart @@ -0,0 +1,39 @@ +import 'package:sqlite_async/src/sqlite_connection.dart'; + +import '../../definitions.dart'; +import './abstract_sqlite_database.dart'; + +class SqliteDatabase extends AbstractSqliteDatabase { + @override + bool get closed => throw UnimplementedError(); + + factory SqliteDatabase( + {required path, + int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + throw UnimplementedError(); + } + + SqliteDatabase.withFactory(SqliteOpenFactory openFactory, + {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + throw UnimplementedError(); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + throw UnimplementedError(); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + throw UnimplementedError(); + } + + @override + Future close() { + // TODO: implement close + throw UnimplementedError(); + } +} diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database.dart new file mode 100644 index 0000000..270f174 --- /dev/null +++ b/lib/src/database/web/web_sqlite_database.dart @@ -0,0 +1,26 @@ +import 'package:sqlite_async/src/sqlite_connection.dart'; + +import '../abstract_sqlite_database.dart'; + +class SqliteDatabase extends AbstractSqliteDatabase { + @override + bool get closed => throw UnimplementedError(); + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + throw UnimplementedError(); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + throw UnimplementedError(); + } + + @override + Future close() { + // TODO: implement close + throw UnimplementedError(); + } +} diff --git a/lib/src/default_sqlite_open_factory.dart b/lib/src/default_sqlite_open_factory.dart deleted file mode 100644 index a8b05ef..0000000 --- a/lib/src/default_sqlite_open_factory.dart +++ /dev/null @@ -1,22 +0,0 @@ -import 'package:sqlite3/sqlite3.dart'; -import 'package:sqlite_async/src/sqlite_open_factory.dart'; -import 'package:sqlite_async/src/sqlite_options.dart'; - -class DefaultSqliteOpenFactory - extends AbstractDefaultSqliteOpenFactory { - const DefaultSqliteOpenFactory( - {required super.path, - super.sqliteOptions = const SqliteOptions.defaults()}); - - @override - Database open(SqliteOpenOptions options) { - final mode = options.openMode; - - var db = sqlite3.open(path, mode: mode, mutex: false); - - for (var statement in pragmaStatements(options)) { - db.execute(statement); - } - return db; - } -} diff --git a/lib/src/mutex.dart b/lib/src/mutex.dart index 5725554..28393a5 100644 --- a/lib/src/mutex.dart +++ b/lib/src/mutex.dart @@ -3,7 +3,7 @@ // (MIT) import 'dart:async'; -import 'port_channel.dart'; +import 'database/native/port_channel.dart'; abstract class Mutex { factory Mutex() { diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/open_factory/abstract_open_factory.dart new file mode 100644 index 0000000..3e59149 --- /dev/null +++ b/lib/src/open_factory/abstract_open_factory.dart @@ -0,0 +1,63 @@ +import 'dart:async'; + +import 'package:sqlite3/common.dart' as sqlite; +import '../../definitions.dart'; + +/// Factory to create new SQLite database connections. +/// +/// Since connections are opened in dedicated background isolates, this class +/// must be safe to pass to different isolates. +abstract class SqliteOpenFactory { + FutureOr open(SqliteOpenOptions options); +} + +class SqliteOpenOptions { + /// Whether this is the primary write connection for the database. + final bool primaryConnection; + + /// Whether this connection is read-only. + final bool readOnly; + + const SqliteOpenOptions( + {required this.primaryConnection, required this.readOnly}); + + sqlite.OpenMode get openMode { + if (primaryConnection) { + return sqlite.OpenMode.readWriteCreate; + } else if (readOnly) { + return sqlite.OpenMode.readOnly; + } else { + return sqlite.OpenMode.readWrite; + } + } +} + +/// The default database factory. +/// +/// This takes care of opening the database, and running PRAGMA statements +/// to configure the connection. +/// +/// Override the [open] method to customize the process. +abstract class AbstractDefaultSqliteOpenFactory + implements SqliteOpenFactory { + final String path; + final SqliteOptions sqliteOptions; + + const AbstractDefaultSqliteOpenFactory( + {required this.path, + this.sqliteOptions = const SqliteOptions.defaults()}); + + List pragmaStatements(SqliteOpenOptions options); + + T openDB(SqliteOpenOptions options); + + @override + T open(SqliteOpenOptions options) { + var db = openDB(options); + + for (var statement in pragmaStatements(options)) { + db.execute(statement); + } + return db; + } +} diff --git a/lib/src/open_factory/native/native_sqlite_open_factory.dart b/lib/src/open_factory/native/native_sqlite_open_factory.dart new file mode 100644 index 0000000..c93f3c6 --- /dev/null +++ b/lib/src/open_factory/native/native_sqlite_open_factory.dart @@ -0,0 +1,38 @@ +import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import '../abstract_open_factory.dart'; + +class DefaultSqliteOpenFactory + extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}); + + @override + Database openDB(SqliteOpenOptions options) { + final mode = options.openMode; + var db = sqlite3.open(path, mode: mode, mutex: false); + return db; + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + List statements = []; + + if (options.primaryConnection && sqliteOptions.journalMode != null) { + // Persisted - only needed on the primary connection + statements + .add('PRAGMA journal_mode = ${sqliteOptions.journalMode!.name}'); + } + if (!options.readOnly && sqliteOptions.journalSizeLimit != null) { + // Needed on every writable connection + statements.add( + 'PRAGMA journal_size_limit = ${sqliteOptions.journalSizeLimit!}'); + } + if (sqliteOptions.synchronous != null) { + // Needed on every connection + statements.add('PRAGMA synchronous = ${sqliteOptions.synchronous!.name}'); + } + return statements; + } +} diff --git a/lib/src/open_factory/stub_sqlite_open_factory.dart b/lib/src/open_factory/stub_sqlite_open_factory.dart new file mode 100644 index 0000000..1dcec61 --- /dev/null +++ b/lib/src/open_factory/stub_sqlite_open_factory.dart @@ -0,0 +1,19 @@ +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/src/sqlite_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; + +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}); + + @override + CommonDatabase openDB(SqliteOpenOptions options) { + throw UnimplementedError(); + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + throw UnimplementedError(); + } +} diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory.dart new file mode 100644 index 0000000..4aa8970 --- /dev/null +++ b/lib/src/open_factory/web/web_sqlite_open_factory.dart @@ -0,0 +1,25 @@ +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite3/wasm.dart'; +import '../abstract_open_factory.dart'; + +class DefaultSqliteOpenFactory + extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}); + + @override + CommonDatabase openDB(SqliteOpenOptions options) { + if (sqliteOptions.wasmSqlite3 == null) { + throw ArgumentError('WASM Sqlite3 implementation was not provided'); + } + + return sqliteOptions.wasmSqlite3!.open("/" + path); + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + // WAL mode is not supported on web + return []; + } +} diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 8738aa1..8394525 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -1,230 +1,9 @@ -import 'dart:async'; -import 'dart:isolate'; - -import 'connection_pool.dart'; -import 'database_utils.dart'; -import 'default_sqlite_open_factory.dart'; -import 'isolate_connection_factory.dart'; -import 'mutex.dart'; -import 'port_channel.dart'; -import 'sqlite_connection.dart'; -import 'sqlite_connection_impl.dart'; -import 'sqlite_open_factory.dart'; -import 'sqlite_options.dart'; -import 'sqlite_queries.dart'; -import 'update_notification.dart'; -import 'package:sqlite3/sqlite3.dart'; - -/// A SQLite database instance. -/// -/// Use one instance per database file. If multiple instances are used, update -/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. -class SqliteDatabase with SqliteQueries implements SqliteConnection { - /// The maximum number of concurrent read transactions if not explicitly specified. - static const int defaultMaxReaders = 5; - - /// Maximum number of concurrent read transactions. - final int maxReaders; - - /// Global lock to serialize write transactions. - final SimpleMutex mutex = SimpleMutex(); - - /// Factory that opens a raw database connection in each isolate. - /// - /// This must be safe to pass to different isolates. - /// - /// Use a custom class for this to customize the open process. - final SqliteOpenFactory openFactory; - - /// Use this stream to subscribe to notifications of updates to tables. - @override - late final Stream updates; - - final StreamController _updatesController = - StreamController.broadcast(); - - late final PortServer _eventsPort; - - late final SqliteConnectionImpl _internalConnection; - late final SqliteConnectionPool _pool; - late final Future _initialized; - - /// Open a SqliteDatabase. - /// - /// Only a single SqliteDatabase per [path] should be opened at a time. - /// - /// A connection pool is used by default, allowing multiple concurrent read - /// transactions, and a single concurrent write transaction. Write transactions - /// do not block read transactions, and read transactions will see the state - /// from the last committed write transaction. - /// - /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabase( - {required path, - int maxReaders = defaultMaxReaders, - SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = - DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); - } - - /// Advanced: Open a database with a specified factory. - /// - /// The factory is used to open each database connection in background isolates. - /// - /// Use when control is required over the opening process. Examples include: - /// 1. Specifying the path to `libsqlite.so` on Linux. - /// 2. Running additional per-connection PRAGMA statements on each connection. - /// 3. Creating custom SQLite functions. - /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(this.openFactory, - {this.maxReaders = defaultMaxReaders}) { - updates = _updatesController.stream; - - _listenForEvents(); - - _internalConnection = _openPrimaryConnection(debugName: 'sqlite-writer'); - _pool = SqliteConnectionPool(openFactory, - upstreamPort: _eventsPort.client(), - updates: updates, - writeConnection: _internalConnection, - debugName: 'sqlite', - maxReaders: maxReaders, - mutex: mutex); - - _initialized = _init(); - } - - Future _init() async { - await _internalConnection.ready; - } - - /// Wait for initialization to complete. - /// - /// While initializing is automatic, this helps to catch and report initialization errors. - Future initialize() async { - await _initialized; - } - - @override - bool get closed { - return _pool.closed; - } - - void _listenForEvents() { - UpdateNotification? updates; - - Map subscriptions = {}; - - _eventsPort = PortServer((message) async { - if (message is UpdateNotification) { - if (updates == null) { - updates = message; - // Use the mutex to only send updates after the current transaction. - // Do take care to avoid getting a lock for each individual update - - // that could add massive performance overhead. - mutex.lock(() async { - if (updates != null) { - _updatesController.add(updates!); - updates = null; - } - }); - } else { - updates!.tables.addAll(message.tables); - } - return null; - } else if (message is InitDb) { - await _initialized; - return null; - } else if (message is SubscribeToUpdates) { - if (subscriptions.containsKey(message.port)) { - return; - } - final subscription = _updatesController.stream.listen((event) { - message.port.send(event); - }); - subscriptions[message.port] = subscription; - return null; - } else if (message is UnsubscribeToUpdates) { - final subscription = subscriptions.remove(message.port); - subscription?.cancel(); - return null; - } else { - throw ArgumentError('Unknown message type: $message'); - } - }); - } - - /// A connection factory that can be passed to different isolates. - /// - /// Use this to access the database in background isolates. - IsolateConnectionFactory isolateConnectionFactory() { - return IsolateConnectionFactory( - openFactory: openFactory, - mutex: mutex.shared, - upstreamPort: _eventsPort.client()); - } - - SqliteConnectionImpl _openPrimaryConnection({String? debugName}) { - return SqliteConnectionImpl( - upstreamPort: _eventsPort.client(), - primary: true, - updates: updates, - debugName: debugName, - mutex: mutex, - readOnly: false, - openFactory: openFactory); - } - - @override - Future close() async { - await _pool.close(); - _updatesController.close(); - _eventsPort.close(); - await mutex.close(); - } - - /// Open a read-only transaction. - /// - /// Up to [maxReaders] read transactions can run concurrently. - /// After that, read transactions are queued. - /// - /// Read transactions can run concurrently to a write transaction. - /// - /// Changes from any write transaction are not visible to read transactions - /// started before it. - @override - Future readTransaction( - Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout}) { - return _pool.readTransaction(callback, lockTimeout: lockTimeout); - } - - /// Open a read-write transaction. - /// - /// Only a single write transaction can run at a time - any concurrent - /// transactions are queued. - /// - /// The write transaction is automatically committed when the callback finishes, - /// or rolled back on any error. - @override - Future writeTransaction( - Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout}) { - return _pool.writeTransaction(callback, lockTimeout: lockTimeout); - } - - @override - Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { - return _pool.readLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); - } - - @override - Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { - return _pool.writeLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); - } -} +// This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an +// To conditionally export an implementation for either web or "native" platforms +// The sqlite library uses dart:ffi which is not supported on web + +export './database/stub_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './database/native/native_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './database/web/web_sqlite_database.dart'; diff --git a/lib/src/sqlite_migrations.dart b/lib/src/sqlite_migrations.dart index fe8e701..a0c9991 100644 --- a/lib/src/sqlite_migrations.dart +++ b/lib/src/sqlite_migrations.dart @@ -1,7 +1,7 @@ import 'dart:async'; import 'dart:convert'; -import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite3/common.dart'; import 'sqlite_connection.dart'; diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 812e822..afc2f2a 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,70 +1,7 @@ -import 'dart:async'; +export './open_factory/abstract_open_factory.dart'; -import 'package:sqlite3/common.dart' as sqlite; - -import 'sqlite_options.dart'; - -/// Factory to create new SQLite database connections. -/// -/// Since connections are opened in dedicated background isolates, this class -/// must be safe to pass to different isolates. -abstract class SqliteOpenFactory { - FutureOr open(SqliteOpenOptions options); -} - -class SqliteOpenOptions { - /// Whether this is the primary write connection for the database. - final bool primaryConnection; - - /// Whether this connection is read-only. - final bool readOnly; - - const SqliteOpenOptions( - {required this.primaryConnection, required this.readOnly}); - - sqlite.OpenMode get openMode { - if (primaryConnection) { - return sqlite.OpenMode.readWriteCreate; - } else if (readOnly) { - return sqlite.OpenMode.readOnly; - } else { - return sqlite.OpenMode.readWrite; - } - } -} - -/// The default database factory. -/// -/// This takes care of opening the database, and running PRAGMA statements -/// to configure the connection. -/// -/// Override the [open] method to customize the process. -abstract class AbstractDefaultSqliteOpenFactory - implements SqliteOpenFactory { - final String path; - final SqliteOptions sqliteOptions; - - const AbstractDefaultSqliteOpenFactory( - {required this.path, - this.sqliteOptions = const SqliteOptions.defaults()}); - - List pragmaStatements(SqliteOpenOptions options) { - List statements = []; - - if (options.primaryConnection && sqliteOptions.journalMode != null) { - // Persisted - only needed on the primary connection - statements - .add('PRAGMA journal_mode = ${sqliteOptions.journalMode!.name}'); - } - if (!options.readOnly && sqliteOptions.journalSizeLimit != null) { - // Needed on every writable connection - statements.add( - 'PRAGMA journal_size_limit = ${sqliteOptions.journalSizeLimit!}'); - } - if (sqliteOptions.synchronous != null) { - // Needed on every connection - statements.add('PRAGMA synchronous = ${sqliteOptions.synchronous!.name}'); - } - return statements; - } -} +export './open_factory/stub_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './open_factory/native/native_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './open_factory/web/web_sqlite_open_factory.dart'; diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index 36beb7c..adb2a9f 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -1,3 +1,5 @@ +import 'package:sqlite3/wasm.dart'; + class SqliteOptions { /// SQLite journal mode. Defaults to [SqliteJournalMode.wal]. final SqliteJournalMode? journalMode; @@ -11,15 +13,28 @@ class SqliteOptions { /// attempt to truncate the file afterwards. final int? journalSizeLimit; + /// The implementation for SQLite + /// This is required for Web WASM + /// final wasmSqlite3 = + /// await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); + /// wasmSqlite3.registerVirtualFileSystem( + /// await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), + /// makeDefault: true, + /// ); + /// Pass the initialized wasmSqlite3 here + final WasmSqlite3? wasmSqlite3; + const SqliteOptions.defaults() : journalMode = SqliteJournalMode.wal, journalSizeLimit = 6 * 1024 * 1024, // 1.5x the default checkpoint size - synchronous = SqliteSynchronous.normal; + synchronous = SqliteSynchronous.normal, + wasmSqlite3 = null; const SqliteOptions( {this.journalMode = SqliteJournalMode.wal, this.journalSizeLimit = 6 * 1024 * 1024, - this.synchronous = SqliteSynchronous.normal}); + this.synchronous = SqliteSynchronous.normal, + this.wasmSqlite3 = null}); } /// SQLite journal mode. Set on the primary connection. diff --git a/lib/src/sqlite_queries.dart b/lib/src/sqlite_queries.dart index 1b53e3c..959a97d 100644 --- a/lib/src/sqlite_queries.dart +++ b/lib/src/sqlite_queries.dart @@ -1,6 +1,6 @@ import 'package:sqlite3/common.dart' as sqlite; -import 'database_utils.dart'; +import 'utils/shared_utils.dart'; import 'sqlite_connection.dart'; import 'update_notification.dart'; diff --git a/lib/src/utils/database_utils.dart b/lib/src/utils/database_utils.dart new file mode 100644 index 0000000..9041e2a --- /dev/null +++ b/lib/src/utils/database_utils.dart @@ -0,0 +1,2 @@ +export 'native_database_utils.dart'; +export 'shared_utils.dart'; diff --git a/lib/src/utils/native_database_utils.dart b/lib/src/utils/native_database_utils.dart new file mode 100644 index 0000000..6b03913 --- /dev/null +++ b/lib/src/utils/native_database_utils.dart @@ -0,0 +1,13 @@ +import 'dart:isolate'; + +class SubscribeToUpdates { + final SendPort port; + + SubscribeToUpdates(this.port); +} + +class UnsubscribeToUpdates { + final SendPort port; + + UnsubscribeToUpdates(this.port); +} diff --git a/lib/src/database_utils.dart b/lib/src/utils/shared_utils.dart similarity index 91% rename from lib/src/database_utils.dart rename to lib/src/utils/shared_utils.dart index b19abda..c911bbc 100644 --- a/lib/src/database_utils.dart +++ b/lib/src/utils/shared_utils.dart @@ -1,8 +1,7 @@ import 'dart:async'; import 'dart:convert'; -import 'dart:isolate'; -import 'sqlite_connection.dart'; +import '../sqlite_connection.dart'; Future internalReadTransaction(SqliteReadContext ctx, Future Function(SqliteReadContext tx) callback) async { @@ -78,18 +77,6 @@ class InitDb { const InitDb(); } -class SubscribeToUpdates { - final SendPort port; - - SubscribeToUpdates(this.port); -} - -class UnsubscribeToUpdates { - final SendPort port; - - UnsubscribeToUpdates(this.port); -} - Object? mapParameter(Object? parameter) { if (parameter == null || parameter is int || diff --git a/lib/utils.dart b/lib/utils.dart new file mode 100644 index 0000000..9883819 --- /dev/null +++ b/lib/utils.dart @@ -0,0 +1 @@ +export 'src/utils.dart'; diff --git a/scripts/benchmark.dart b/scripts/benchmark.dart index 10a133e..11df819 100644 --- a/scripts/benchmark.dart +++ b/scripts/benchmark.dart @@ -4,8 +4,8 @@ import 'dart:math'; import 'package:benchmarking/benchmarking.dart'; import 'package:collection/collection.dart'; - -import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/database/native/native_sqlite_database.dart'; import '../test/util.dart'; @@ -24,7 +24,7 @@ class SqliteBenchmark { List benchmarks = [ SqliteBenchmark('Insert: JSON1', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { for (var i = 0; i < parameters.length; i += 5000) { var sublist = parameters.sublist(i, min(parameters.length, i + 5000)); @@ -36,7 +36,7 @@ List benchmarks = [ }); }, maxBatchSize: 20000), SqliteBenchmark('Read: JSON1', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.readTransaction((tx) async { for (var i = 0; i < parameters.length; i += 10000) { var sublist = List.generate(10000, (index) => index); @@ -59,26 +59,26 @@ List benchmarks = [ }); }, maxBatchSize: 10000, enabled: true), SqliteBenchmark('Write lock', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { for (var _ in parameters) { await db.writeLock((tx) async {}); } }, maxBatchSize: 5000, enabled: false), SqliteBenchmark('Read lock', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { for (var _ in parameters) { await db.readLock((tx) async {}); } }, maxBatchSize: 5000, enabled: false), SqliteBenchmark('Insert: Direct', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { for (var params in parameters) { await db.execute( 'INSERT INTO customers(name, email) VALUES(?, ?)', params); } }, maxBatchSize: 500), SqliteBenchmark('Insert: writeTransaction', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { for (var params in parameters) { await tx.execute( @@ -109,7 +109,7 @@ List benchmarks = [ }); }, maxBatchSize: 2000), SqliteBenchmark('Insert: writeTransaction no await', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { for (var params in parameters) { tx.execute('INSERT INTO customers(name, email) VALUES(?, ?)', params); @@ -117,7 +117,7 @@ List benchmarks = [ }); }, maxBatchSize: 1000), SqliteBenchmark('Insert: computeWithDatabase', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.computeWithDatabase((db) async { for (var params in parameters) { db.execute('INSERT INTO customers(name, email) VALUES(?, ?)', params); @@ -125,7 +125,7 @@ List benchmarks = [ }); }), SqliteBenchmark('Insert: computeWithDatabase, prepared', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.computeWithDatabase((db) async { var stmt = db.prepare('INSERT INTO customers(name, email) VALUES(?, ?)'); try { @@ -138,14 +138,14 @@ List benchmarks = [ }); }), SqliteBenchmark('Insert: executeBatch', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { await tx.executeBatch( 'INSERT INTO customers(name, email) VALUES(?, ?)', parameters); }); }), SqliteBenchmark('Insert: computeWithDatabase, prepared x10', - (SqliteDatabase db, List> parameters) async { + (AbstractSqliteDatabase db, List> parameters) async { await db.computeWithDatabase((db) async { var stmt = db.prepare( 'INSERT INTO customers(name, email) VALUES (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?)'); @@ -166,7 +166,7 @@ void main() async { var parameters = List.generate( 20000, (index) => ['Test user $index', 'user$index@example.org']); - createTables(SqliteDatabase db) async { + createTables(AbstractSqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute('DROP TABLE IF EXISTS customers'); await tx.execute( diff --git a/test/basic_test.dart b/test/basic_test.dart index ba15a18..eedb9ee 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -4,6 +4,7 @@ import 'dart:math'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; @@ -21,7 +22,7 @@ void main() { await cleanDb(path: path); }); - createTables(SqliteDatabase db) async { + createTables(AbstractSqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE test_data(id INTEGER PRIMARY KEY AUTOINCREMENT, description TEXT)'); diff --git a/test/close_test.dart b/test/close_test.dart index be3e933..d29519f 100644 --- a/test/close_test.dart +++ b/test/close_test.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; @@ -18,7 +19,7 @@ void main() { await cleanDb(path: path); }); - createTables(SqliteDatabase db) async { + createTables(AbstractSqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE test_data(id INTEGER PRIMARY KEY AUTOINCREMENT, description TEXT)'); diff --git a/test/json1_test.dart b/test/json1_test.dart index 62bd6c3..5aa56a1 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -1,4 +1,5 @@ import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; @@ -32,7 +33,7 @@ void main() { await cleanDb(path: path); }); - createTables(SqliteDatabase db) async { + createTables(AbstractSqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE users(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, email TEXT)'); diff --git a/test/util.dart b/test/util.dart index 55e8367..1a1650a 100644 --- a/test/util.dart +++ b/test/util.dart @@ -6,7 +6,10 @@ import 'package:glob/glob.dart'; import 'package:glob/list_local_fs.dart'; import 'package:sqlite3/open.dart' as sqlite_open; import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/open_factory/abstract_open_factory.dart'; +import 'package:sqlite_async/src/database/native/native_sqlite_database.dart'; +import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory.dart'; import 'package:test_api/src/backend/invoker.dart'; const defaultSqlitePath = 'libsqlite3.so.0'; diff --git a/test/watch_test.dart b/test/watch_test.dart index f102001..d8a8436 100644 --- a/test/watch_test.dart +++ b/test/watch_test.dart @@ -4,13 +4,15 @@ import 'dart:math'; import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database_utils.dart'; +import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/database/native/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:test/test.dart'; import 'util.dart'; void main() { - createTables(SqliteDatabase db) async { + createTables(AbstractSqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); From 55252e481c0282ac677e6fc72ecf2269e0499f10 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 11 Jan 2024 09:20:48 +0200 Subject: [PATCH 06/57] abstract factories. Fix tests --- lib/sqlite_async.dart | 1 + .../database/abstract_sqlite_database.dart | 10 ++- .../native/native_sqlite_database.dart | 4 +- lib/src/database/sqlite_database_adapter.dart | 5 ++ lib/src/database/stub_sqlite_database.dart | 11 ++- lib/src/database/web/web_sqlite_database.dart | 47 +++++++++++ lib/src/isolate_connection_factory.dart | 30 +++++++ .../abstract_isolate_connection_factory.dart | 26 ++++++ .../web_isolate_connection_factory.dart | 37 +++++++++ .../stub_isolate_connection_factory.dart | 32 ++++++++ .../native_isolate_connection_factory.dart} | 30 +++---- lib/src/sqlite_database.dart | 82 +++++++++++++++++-- lib/src/sqlite_open_factory.dart | 27 +++++- scripts/benchmark.dart | 14 ++-- test/basic_test.dart | 3 +- test/isolate_test.dart | 3 +- test/util.dart | 10 +-- test/watch_test.dart | 7 +- 18 files changed, 323 insertions(+), 56 deletions(-) create mode 100644 lib/src/database/sqlite_database_adapter.dart create mode 100644 lib/src/isolate_connection_factory.dart create mode 100644 lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart create mode 100644 lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart create mode 100644 lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart rename lib/src/{database/native/isolate_connection_factory.dart => isolate_connection_factory/web/native_isolate_connection_factory.dart} (72%) diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index 5e51c89..a354c3a 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -3,6 +3,7 @@ /// See [SqliteDatabase] as a starting point. library; +export 'src/isolate_connection_factory.dart'; export 'src/sqlite_connection.dart'; export 'src/sqlite_database.dart'; export 'src/sqlite_migrations.dart'; diff --git a/lib/src/database/abstract_sqlite_database.dart b/lib/src/database/abstract_sqlite_database.dart index 26d59ff..ceb1c27 100644 --- a/lib/src/database/abstract_sqlite_database.dart +++ b/lib/src/database/abstract_sqlite_database.dart @@ -1,9 +1,8 @@ import 'dart:async'; +import 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; + import '../../definitions.dart'; -import '../sqlite_connection.dart'; -import '../sqlite_queries.dart'; -import '../update_notification.dart'; /// A SQLite database instance. /// @@ -43,6 +42,11 @@ abstract class AbstractSqliteDatabase await _initialized; } + /// A connection factory that can be passed to different isolates. + /// + /// Use this to access the database in background isolates. + AbstractIsolateConnectionFactory isolateConnectionFactory(); + /// Open a read-only transaction. /// /// Up to [maxReaders] read transactions can run concurrently. diff --git a/lib/src/database/native/native_sqlite_database.dart b/lib/src/database/native/native_sqlite_database.dart index 646bc84..dfc8bda 100644 --- a/lib/src/database/native/native_sqlite_database.dart +++ b/lib/src/database/native/native_sqlite_database.dart @@ -14,7 +14,7 @@ import '../abstract_sqlite_database.dart'; import 'port_channel.dart'; import 'connection_pool.dart'; import 'sqlite_connection_impl.dart'; -import 'isolate_connection_factory.dart'; +import '../../isolate_connection_factory/web/native_isolate_connection_factory.dart'; /// A SQLite database instance. /// @@ -25,7 +25,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override late final Stream updates; - final SqliteOpenFactory openFactory; + final AbstractDefaultSqliteOpenFactory openFactory; final StreamController _updatesController = StreamController.broadcast(); diff --git a/lib/src/database/sqlite_database_adapter.dart b/lib/src/database/sqlite_database_adapter.dart new file mode 100644 index 0000000..9336dbf --- /dev/null +++ b/lib/src/database/sqlite_database_adapter.dart @@ -0,0 +1,5 @@ +export 'stub_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './native/native_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './web/web_sqlite_database.dart'; diff --git a/lib/src/database/stub_sqlite_database.dart b/lib/src/database/stub_sqlite_database.dart index c58ee12..aa4a8c6 100644 --- a/lib/src/database/stub_sqlite_database.dart +++ b/lib/src/database/stub_sqlite_database.dart @@ -1,7 +1,4 @@ -import 'package:sqlite_async/src/sqlite_connection.dart'; - -import '../../definitions.dart'; -import './abstract_sqlite_database.dart'; +import 'package:sqlite_async/sqlite_async.dart'; class SqliteDatabase extends AbstractSqliteDatabase { @override @@ -36,4 +33,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { // TODO: implement close throw UnimplementedError(); } + + @override + IsolateConnectionFactory isolateConnectionFactory() { + // TODO: implement isolateConnectionFactory + throw UnimplementedError(); + } } diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database.dart index 270f174..a69c6ac 100644 --- a/lib/src/database/web/web_sqlite_database.dart +++ b/lib/src/database/web/web_sqlite_database.dart @@ -1,3 +1,5 @@ +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import '../abstract_sqlite_database.dart'; @@ -6,6 +8,45 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); + late final CommonDatabase con; + + // late final Future _initialized; + + /// Open a SqliteDatabase. + /// + /// Only a single SqliteDatabase per [path] should be opened at a time. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + factory SqliteDatabase( + {required path, + int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + final factory = + DefaultSqliteOpenFactory(path: path, sqliteOptions: options); + return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + } + + /// Advanced: Open a database with a specified factory. + /// + /// The factory is used to open each database connection in background isolates. + /// + /// Use when control is required over the opening process. Examples include: + /// 1. Specifying the path to `libsqlite.so` on Linux. + /// 2. Running additional per-connection PRAGMA statements on each connection. + /// 3. Creating custom SQLite functions. + /// 4. Creating temporary views or triggers. + SqliteDatabase.withFactory(AbstractDefaultSqliteOpenFactory openFactory, + {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + super.openFactory = openFactory; + super.maxReaders = maxReaders; + // con = openFactory.open(options) + } + @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) { @@ -23,4 +64,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { // TODO: implement close throw UnimplementedError(); } + + @override + IsolateConnectionFactory isolateConnectionFactory() { + // TODO: implement isolateConnectionFactory + throw UnimplementedError(); + } } diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/isolate_connection_factory.dart new file mode 100644 index 0000000..123a2a5 --- /dev/null +++ b/lib/src/isolate_connection_factory.dart @@ -0,0 +1,30 @@ +// This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an +// To conditionally export an implementation for either web or "native" platforms +// The sqlite library uses dart:ffi which is not supported on web + +import 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; +export 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; + +import '../definitions.dart'; +import './isolate_connection_factory/stub_isolate_connection_factory.dart' as base + if (dart.library.io) './isolate_connection_factory/native/isolate_connection_factory.dart' + if (dart.library.html) './isolate_connection_factory/web/isolate_connection_factory.dart'; + + +class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + late AbstractIsolateConnectionFactory adapter; + + IsolateConnectionFactory({ + required AbstractDefaultSqliteOpenFactory openFactory, + }) { + super.openFactory = openFactory; + adapter = base.IsolateConnectionFactory(openFactory: openFactory); + } + + + @override + SqliteConnection open({String? debugName, bool readOnly = false}) { + return adapter.open(debugName: debugName, readOnly: readOnly); + } + +} \ No newline at end of file diff --git a/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart new file mode 100644 index 0000000..7f3febb --- /dev/null +++ b/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart @@ -0,0 +1,26 @@ +import 'dart:async'; +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/definitions.dart'; + +/// A connection factory that can be passed to different isolates. +abstract class AbstractIsolateConnectionFactory { + late SqliteOpenFactory openFactory; + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + SqliteConnection open({String? debugName, bool readOnly = false}); + + /// Opens a synchronous sqlite.Database directly in the current isolate. + /// + /// This gives direct access to the database, but: + /// 1. No app-level locking is performed automatically. Transactions may fail + /// with SQLITE_BUSY if another isolate is using the database at the same time. + /// 2. Other connections are not notified of any updates to tables made within + /// this connection. + Future openRawDatabase({bool readOnly = false}) async { + final db = await openFactory + .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); + return db; + } +} diff --git a/lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart new file mode 100644 index 0000000..b90a42d --- /dev/null +++ b/lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart @@ -0,0 +1,37 @@ +import 'dart:async'; + +import 'package:sqlite3/common.dart'; + +import '../../sqlite_connection.dart'; +import '../../sqlite_open_factory.dart'; +import '../abstract_isolate_connection_factory.dart'; + +/// A connection factory that can be passed to different isolates. +class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + IsolateConnectionFactory({ + required AbstractDefaultSqliteOpenFactory openFactory, + }) { + super.openFactory = openFactory; + } + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + SqliteConnection open({String? debugName, bool readOnly = false}) { + // TODO + return {} as SqliteConnection; + } + + /// Opens a synchronous sqlite.Database directly in the current isolate. + /// + /// This gives direct access to the database, but: + /// 1. No app-level locking is performed automatically. Transactions may fail + /// with SQLITE_BUSY if another isolate is using the database at the same time. + /// 2. Other connections are not notified of any updates to tables made within + /// this connection. + Future openRawDatabase({bool readOnly = false}) async { + final db = await openFactory + .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); + return db; + } +} diff --git a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart new file mode 100644 index 0000000..73a5c38 --- /dev/null +++ b/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart @@ -0,0 +1,32 @@ +import 'dart:async'; + +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/definitions.dart'; +import 'abstract_isolate_connection_factory.dart'; + +/// A connection factory that can be passed to different isolates. +class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + IsolateConnectionFactory({ + required AbstractDefaultSqliteOpenFactory openFactory, + }) { + super.openFactory = openFactory; + } + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + SqliteConnection open({String? debugName, bool readOnly = false}) { + throw UnimplementedError(); + } + + /// Opens a synchronous sqlite.Database directly in the current isolate. + /// + /// This gives direct access to the database, but: + /// 1. No app-level locking is performed automatically. Transactions may fail + /// with SQLITE_BUSY if another isolate is using the database at the same time. + /// 2. Other connections are not notified of any updates to tables made within + /// this connection. + Future openRawDatabase({bool readOnly = false}) async { + throw UnimplementedError(); + } +} diff --git a/lib/src/database/native/isolate_connection_factory.dart b/lib/src/isolate_connection_factory/web/native_isolate_connection_factory.dart similarity index 72% rename from lib/src/database/native/isolate_connection_factory.dart rename to lib/src/isolate_connection_factory/web/native_isolate_connection_factory.dart index 34ac8c2..ffb76c2 100644 --- a/lib/src/database/native/isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/web/native_isolate_connection_factory.dart @@ -8,19 +8,21 @@ import '../../sqlite_connection.dart'; import '../../sqlite_open_factory.dart'; import '../../update_notification.dart'; import '../../utils/native_database_utils.dart'; -import 'port_channel.dart'; -import 'sqlite_connection_impl.dart'; +import '../../database/native/port_channel.dart'; +import '../../database/native/sqlite_connection_impl.dart'; +import '../abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. -class IsolateConnectionFactory { - SqliteOpenFactory openFactory; +class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { SerializedMutex mutex; SerializedPortClient upstreamPort; IsolateConnectionFactory( - {required this.openFactory, + {required AbstractDefaultSqliteOpenFactory openFactory, required this.mutex, - required this.upstreamPort}); + required this.upstreamPort}) { + super.openFactory = openFactory; + } /// Open a new SqliteConnection. /// @@ -31,7 +33,8 @@ class IsolateConnectionFactory { var openMutex = mutex.open(); return _IsolateSqliteConnection( - openFactory: openFactory, + openFactory: + openFactory as AbstractDefaultSqliteOpenFactory, mutex: openMutex, upstreamPort: upstreamPort, readOnly: readOnly, @@ -42,19 +45,6 @@ class IsolateConnectionFactory { updates.close(); }); } - - /// Opens a synchronous sqlite.Database directly in the current isolate. - /// - /// This gives direct access to the database, but: - /// 1. No app-level locking is performed automatically. Transactions may fail - /// with SQLITE_BUSY if another isolate is using the database at the same time. - /// 2. Other connections are not notified of any updates to tables made within - /// this connection. - Future openRawDatabase({bool readOnly = false}) async { - final db = await openFactory - .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); - return db; - } } class _IsolateUpdateListener { diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 8394525..12d76a5 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -2,8 +2,80 @@ // To conditionally export an implementation for either web or "native" platforms // The sqlite library uses dart:ffi which is not supported on web -export './database/stub_sqlite_database.dart' - // ignore: uri_does_not_exist - if (dart.library.io) './database/native/native_sqlite_database.dart' - // ignore: uri_does_not_exist - if (dart.library.html) './database/web/web_sqlite_database.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +export 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; +import './database/sqlite_database_adapter.dart' as base; + +class SqliteDatabase extends AbstractSqliteDatabase { + static const int defaultMaxReaders = AbstractSqliteDatabase.defaultMaxReaders; + + /// Use this stream to subscribe to notifications of updates to tables. + @override + late final Stream updates; + + late AbstractSqliteDatabase adapter; + + /// Open a SqliteDatabase. + /// + /// Only a single SqliteDatabase per [path] should be opened at a time. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. + SqliteDatabase( + {required path, + int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + final factory = + DefaultSqliteOpenFactory(path: path, sqliteOptions: options); + adapter = base.SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + updates = adapter.updates; + } + + /// Advanced: Open a database with a specified factory. + /// + /// The factory is used to open each database connection in background isolates. + /// + /// Use when control is required over the opening process. Examples include: + /// 1. Specifying the path to `libsqlite.so` on Linux. + /// 2. Running additional per-connection PRAGMA statements on each connection. + /// 3. Creating custom SQLite functions. + /// 4. Creating temporary views or triggers. + SqliteDatabase.withFactory(SqliteOpenFactory openFactory, + {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + super.maxReaders = maxReaders; + adapter = + base.SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); + updates = adapter.updates; + } + + @override + Future close() { + return adapter.close(); + } + + @override + bool get closed => adapter.closed; + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return adapter.readLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return adapter.writeLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); + } + + @override + AbstractIsolateConnectionFactory isolateConnectionFactory() { + return adapter.isolateConnectionFactory(); + } +} diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index afc2f2a..d9fc318 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,7 +1,28 @@ export './open_factory/abstract_open_factory.dart'; -export './open_factory/stub_sqlite_open_factory.dart' - // ignore: uri_does_not_exist +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/definitions.dart'; + +import './open_factory/stub_sqlite_open_factory.dart' as base if (dart.library.io) './open_factory/native/native_sqlite_open_factory.dart' - // ignore: uri_does_not_exist if (dart.library.html) './open_factory/web/web_sqlite_open_factory.dart'; + +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { + late AbstractDefaultSqliteOpenFactory adapter; + + DefaultSqliteOpenFactory( + {required super.path, + super.sqliteOptions = const SqliteOptions.defaults()}) { + adapter = base.DefaultSqliteOpenFactory(path: path, sqliteOptions: super.sqliteOptions) as AbstractDefaultSqliteOpenFactory; + } + + @override + T openDB(SqliteOpenOptions options) { + return adapter.openDB(options); + } + + @override + List pragmaStatements(SqliteOpenOptions options) { + return adapter.pragmaStatements(options); + } +} \ No newline at end of file diff --git a/scripts/benchmark.dart b/scripts/benchmark.dart index 11df819..6e8b3d8 100644 --- a/scripts/benchmark.dart +++ b/scripts/benchmark.dart @@ -4,8 +4,9 @@ import 'dart:math'; import 'package:benchmarking/benchmarking.dart'; import 'package:collection/collection.dart'; -import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; -import 'package:sqlite_async/src/database/native/native_sqlite_database.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/native/native_sqlite_database.dart' + as native_sqlite_database; import '../test/util.dart'; @@ -48,7 +49,8 @@ List benchmarks = [ }, maxBatchSize: 200000, enabled: false), SqliteBenchmark('writeLock in isolate', (SqliteDatabase db, List> parameters) async { - var factory = db.isolateConnectionFactory(); + var factory = (db as native_sqlite_database.SqliteDatabase) + .isolateConnectionFactory(); var len = parameters.length; await Isolate.run(() async { final db = factory.open(); @@ -88,7 +90,8 @@ List benchmarks = [ }, maxBatchSize: 1000), SqliteBenchmark('Insert: executeBatch in isolate', (SqliteDatabase db, List> parameters) async { - var factory = db.isolateConnectionFactory(); + var factory = (db as native_sqlite_database.SqliteDatabase) + .isolateConnectionFactory(); await Isolate.run(() async { final db = factory.open(); await db.executeBatch( @@ -98,7 +101,8 @@ List benchmarks = [ }, maxBatchSize: 20000, enabled: true), SqliteBenchmark('Insert: direct write in isolate', (SqliteDatabase db, List> parameters) async { - var factory = db.isolateConnectionFactory(); + var factory = (db as native_sqlite_database.SqliteDatabase) + .isolateConnectionFactory(); await Isolate.run(() async { final db = factory.open(); for (var params in parameters) { diff --git a/test/basic_test.dart b/test/basic_test.dart index eedb9ee..ba15a18 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -4,7 +4,6 @@ import 'dart:math'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; @@ -22,7 +21,7 @@ void main() { await cleanDb(path: path); }); - createTables(AbstractSqliteDatabase db) async { + createTables(SqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE test_data(id INTEGER PRIMARY KEY AUTOINCREMENT, description TEXT)'); diff --git a/test/isolate_test.dart b/test/isolate_test.dart index 69b5035..ca71e7a 100644 --- a/test/isolate_test.dart +++ b/test/isolate_test.dart @@ -1,6 +1,7 @@ import 'dart:isolate'; import 'package:test/test.dart'; +import 'package:sqlite_async/src/database/native/native_sqlite_database.dart'; import 'util.dart'; @@ -18,7 +19,7 @@ void main() { }); test('Basic Isolate usage', () async { - final db = await setupDatabase(path: path); + final db = await setupDatabase(path: path) as SqliteDatabase; final factory = db.isolateConnectionFactory(); final result = await Isolate.run(() async { diff --git a/test/util.dart b/test/util.dart index 1a1650a..29662ed 100644 --- a/test/util.dart +++ b/test/util.dart @@ -4,12 +4,10 @@ import 'dart:isolate'; import 'package:glob/glob.dart'; import 'package:glob/list_local_fs.dart'; +import 'package:sqlite3/common.dart'; import 'package:sqlite3/open.dart' as sqlite_open; import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; -import 'package:sqlite_async/src/open_factory/abstract_open_factory.dart'; -import 'package:sqlite_async/src/database/native/native_sqlite_database.dart'; -import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory.dart'; +import 'package:sqlite_async/sqlite_async.dart'; import 'package:test_api/src/backend/invoker.dart'; const defaultSqlitePath = 'libsqlite3.so.0'; @@ -24,7 +22,7 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { this.sqlitePath = defaultSqlitePath}); @override - sqlite.Database open(SqliteOpenOptions options) { + CommonDatabase open(SqliteOpenOptions options) { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open(sqlitePath); }); @@ -52,7 +50,7 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { } } -SqliteOpenFactory testFactory({String? path}) { +SqliteOpenFactory testFactory({String? path}) { return TestSqliteOpenFactory(path: path ?? dbPath()); } diff --git a/test/watch_test.dart b/test/watch_test.dart index d8a8436..668a9eb 100644 --- a/test/watch_test.dart +++ b/test/watch_test.dart @@ -4,11 +4,8 @@ import 'dart:math'; import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; -import 'package:sqlite_async/src/database/native/isolate_connection_factory.dart'; -import 'package:sqlite_async/src/utils/shared_utils.dart'; +import 'package:sqlite_async/src/utils/database_utils.dart'; import 'package:test/test.dart'; - import 'util.dart'; void main() { @@ -316,7 +313,7 @@ void main() { }); } -Future> inIsolateWatch(IsolateConnectionFactory factory, +Future> inIsolateWatch(AbstractIsolateConnectionFactory factory, int numberOfQueries, Duration throttleDuration) async { return await Isolate.run(() async { final db = factory.open(); From e54f9bb75841f9b1270f99d46f04bceb4f1840c7 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 11 Jan 2024 10:21:25 +0200 Subject: [PATCH 07/57] cleanup isolates --- .../database/abstract_sqlite_database.dart | 8 ++--- .../native/native_sqlite_database.dart | 32 +++++-------------- lib/src/database/web/web_sqlite_database.dart | 8 +++-- lib/src/isolate_connection_factory.dart | 2 +- .../native_isolate_connection_factory.dart | 2 +- .../stub_isolate_connection_factory.dart | 2 +- .../web_isolate_connection_factory.dart | 2 +- lib/src/sqlite_database.dart | 1 + 8 files changed, 22 insertions(+), 35 deletions(-) rename lib/src/isolate_connection_factory/{web => native}/native_isolate_connection_factory.dart (97%) rename lib/src/isolate_connection_factory/{native => web}/web_isolate_connection_factory.dart (95%) diff --git a/lib/src/database/abstract_sqlite_database.dart b/lib/src/database/abstract_sqlite_database.dart index ceb1c27..01e247d 100644 --- a/lib/src/database/abstract_sqlite_database.dart +++ b/lib/src/database/abstract_sqlite_database.dart @@ -28,18 +28,16 @@ abstract class AbstractSqliteDatabase @override late final Stream updates; - final StreamController _updatesController = + final StreamController updatesController = StreamController.broadcast(); - late final Future _initialized; - - Future _init(); + late final Future isInitialized; /// Wait for initialization to complete. /// /// While initializing is automatic, this helps to catch and report initialization errors. Future initialize() async { - await _initialized; + await isInitialized; } /// A connection factory that can be passed to different isolates. diff --git a/lib/src/database/native/native_sqlite_database.dart b/lib/src/database/native/native_sqlite_database.dart index dfc8bda..09cf844 100644 --- a/lib/src/database/native/native_sqlite_database.dart +++ b/lib/src/database/native/native_sqlite_database.dart @@ -7,6 +7,7 @@ import '../../../mutex.dart'; import '../../utils/database_utils.dart'; import '../../sqlite_connection.dart'; import '../../open_factory/native/native_sqlite_open_factory.dart'; +import '../../isolate_connection_factory/native/native_isolate_connection_factory.dart'; import '../../open_factory/abstract_open_factory.dart'; import '../../sqlite_options.dart'; import '../../update_notification.dart'; @@ -14,27 +15,19 @@ import '../abstract_sqlite_database.dart'; import 'port_channel.dart'; import 'connection_pool.dart'; import 'sqlite_connection_impl.dart'; -import '../../isolate_connection_factory/web/native_isolate_connection_factory.dart'; /// A SQLite database instance. /// /// Use one instance per database file. If multiple instances are used, update /// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. class SqliteDatabase extends AbstractSqliteDatabase { - /// Use this stream to subscribe to notifications of updates to tables. @override - late final Stream updates; - - final AbstractDefaultSqliteOpenFactory openFactory; - - final StreamController _updatesController = - StreamController.broadcast(); + final SqliteOpenFactory openFactory; late final PortServer _eventsPort; late final SqliteConnectionImpl _internalConnection; late final SqliteConnectionPool _pool; - late final Future _initialized; /// Global lock to serialize write transactions. final SimpleMutex mutex = SimpleMutex(); @@ -69,9 +62,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 4. Creating temporary views or triggers. SqliteDatabase.withFactory(this.openFactory, {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { - updates = _updatesController.stream; - - super.maxReaders = maxReaders; + updates = updatesController.stream; _listenForEvents(); @@ -84,20 +75,13 @@ class SqliteDatabase extends AbstractSqliteDatabase { maxReaders: maxReaders, mutex: mutex); - _initialized = _init(); + isInitialized = _init(); } Future _init() async { await _internalConnection.ready; } - /// Wait for initialization to complete. - /// - /// While initializing is automatic, this helps to catch and report initialization errors. - Future initialize() async { - await _initialized; - } - @override bool get closed { return _pool.closed; @@ -117,7 +101,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { // that could add massive performance overhead. mutex.lock(() async { if (updates != null) { - _updatesController.add(updates!); + updatesController.add(updates!); updates = null; } }); @@ -126,13 +110,13 @@ class SqliteDatabase extends AbstractSqliteDatabase { } return null; } else if (message is InitDb) { - await _initialized; + await isInitialized; return null; } else if (message is SubscribeToUpdates) { if (subscriptions.containsKey(message.port)) { return; } - final subscription = _updatesController.stream.listen((event) { + final subscription = updatesController.stream.listen((event) { message.port.send(event); }); subscriptions[message.port] = subscription; @@ -171,7 +155,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override Future close() async { await _pool.close(); - _updatesController.close(); + updatesController.close(); _eventsPort.close(); await mutex.close(); } diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database.dart index a69c6ac..76a7c92 100644 --- a/lib/src/database/web/web_sqlite_database.dart +++ b/lib/src/database/web/web_sqlite_database.dart @@ -8,7 +8,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); - late final CommonDatabase con; + // late final CommonDatabase con; // late final Future _initialized; @@ -40,13 +40,17 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(AbstractDefaultSqliteOpenFactory openFactory, + SqliteDatabase.withFactory(SqliteOpenFactory openFactory, {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { super.openFactory = openFactory; super.maxReaders = maxReaders; + updates = updatesController.stream; + isInitialized = _init(); // con = openFactory.open(options) } + Future _init() async {} + @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) { diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/isolate_connection_factory.dart index 123a2a5..c108194 100644 --- a/lib/src/isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory.dart @@ -15,7 +15,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { late AbstractIsolateConnectionFactory adapter; IsolateConnectionFactory({ - required AbstractDefaultSqliteOpenFactory openFactory, + required SqliteOpenFactory openFactory, }) { super.openFactory = openFactory; adapter = base.IsolateConnectionFactory(openFactory: openFactory); diff --git a/lib/src/isolate_connection_factory/web/native_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart similarity index 97% rename from lib/src/isolate_connection_factory/web/native_isolate_connection_factory.dart rename to lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart index ffb76c2..282553d 100644 --- a/lib/src/isolate_connection_factory/web/native_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart @@ -18,7 +18,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { SerializedPortClient upstreamPort; IsolateConnectionFactory( - {required AbstractDefaultSqliteOpenFactory openFactory, + {required SqliteOpenFactory openFactory, required this.mutex, required this.upstreamPort}) { super.openFactory = openFactory; diff --git a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart index 73a5c38..24d361c 100644 --- a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart @@ -7,7 +7,7 @@ import 'abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { IsolateConnectionFactory({ - required AbstractDefaultSqliteOpenFactory openFactory, + required SqliteOpenFactory openFactory, }) { super.openFactory = openFactory; } diff --git a/lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart similarity index 95% rename from lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart rename to lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart index b90a42d..dc615f5 100644 --- a/lib/src/isolate_connection_factory/native/web_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart @@ -9,7 +9,7 @@ import '../abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { IsolateConnectionFactory({ - required AbstractDefaultSqliteOpenFactory openFactory, + required SqliteOpenFactory openFactory, }) { super.openFactory = openFactory; } diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 12d76a5..3a4823d 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -49,6 +49,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { super.maxReaders = maxReaders; adapter = base.SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); + isInitialized = adapter.isInitialized; updates = adapter.updates; } From e011bd09358144ead7a69c1356241f251965fecd Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 11 Jan 2024 11:17:03 +0200 Subject: [PATCH 08/57] loading WASM implementation --- example/custom_functions_example.dart | 5 +++-- example/linux_cli_example.dart | 3 ++- lib/src/database/web/web_sqlite_database.dart | 14 ++++++++----- .../open_factory/abstract_open_factory.dart | 6 +++--- .../open_factory/open_factory_adapter.dart | 5 +++++ .../web/web_sqlite_open_factory.dart | 9 ++++---- lib/src/sqlite_open_factory.dart | 19 ++++++++++------- lib/src/sqlite_options.dart | 21 ++++++++++++++++--- test/util.dart | 5 +++-- 9 files changed, 59 insertions(+), 28 deletions(-) create mode 100644 lib/src/open_factory/open_factory_adapter.dart diff --git a/example/custom_functions_example.dart b/example/custom_functions_example.dart index c2e9c9d..e451927 100644 --- a/example/custom_functions_example.dart +++ b/example/custom_functions_example.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:io'; import 'dart:isolate'; @@ -10,8 +11,8 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { TestOpenFactory({required super.path, super.sqliteOptions}); @override - CommonDatabase open(SqliteOpenOptions options) { - final db = super.open(options); + FutureOr open(SqliteOpenOptions options) async { + final db = await super.open(options); db.createFunction( functionName: 'sleep', diff --git a/example/linux_cli_example.dart b/example/linux_cli_example.dart index 4a3462c..8d5aa43 100644 --- a/example/linux_cli_example.dart +++ b/example/linux_cli_example.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:ffi'; import 'package:sqlite3/common.dart'; @@ -16,7 +17,7 @@ class TestOpenFactory extends DefaultSqliteOpenFactory { this.sqlitePath = defaultSqlitePath}); @override - CommonDatabase open(SqliteOpenOptions options) { + FutureOr open(SqliteOpenOptions options) async { // For details, see: // https://pub.dev/packages/sqlite3#manually-providing-sqlite3-libraries sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database.dart index 76a7c92..3b36bf8 100644 --- a/lib/src/database/web/web_sqlite_database.dart +++ b/lib/src/database/web/web_sqlite_database.dart @@ -8,7 +8,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); - // late final CommonDatabase con; + late final CommonDatabase con; // late final Future _initialized; @@ -46,20 +46,24 @@ class SqliteDatabase extends AbstractSqliteDatabase { super.maxReaders = maxReaders; updates = updatesController.stream; isInitialized = _init(); - // con = openFactory.open(options) } - Future _init() async {} + Future _init() async { + con = await openFactory + .open(SqliteOpenOptions(primaryConnection: true, readOnly: false)); + } @override Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { + {Duration? lockTimeout, String? debugContext}) async { + await isInitialized; throw UnimplementedError(); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { + {Duration? lockTimeout, String? debugContext}) async { + await isInitialized; throw UnimplementedError(); } diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/open_factory/abstract_open_factory.dart index 3e59149..2adec66 100644 --- a/lib/src/open_factory/abstract_open_factory.dart +++ b/lib/src/open_factory/abstract_open_factory.dart @@ -49,11 +49,11 @@ abstract class AbstractDefaultSqliteOpenFactory List pragmaStatements(SqliteOpenOptions options); - T openDB(SqliteOpenOptions options); + FutureOr openDB(SqliteOpenOptions options); @override - T open(SqliteOpenOptions options) { - var db = openDB(options); + FutureOr open(SqliteOpenOptions options) async { + var db = await openDB(options); for (var statement in pragmaStatements(options)) { db.execute(statement); diff --git a/lib/src/open_factory/open_factory_adapter.dart b/lib/src/open_factory/open_factory_adapter.dart new file mode 100644 index 0000000..af03ec2 --- /dev/null +++ b/lib/src/open_factory/open_factory_adapter.dart @@ -0,0 +1,5 @@ +export 'stub_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './native/native_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './web/web_sqlite_open_factory.dart'; diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory.dart index 4aa8970..986c5e5 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory.dart @@ -9,17 +9,18 @@ class DefaultSqliteOpenFactory super.sqliteOptions = const SqliteOptions.defaults()}); @override - CommonDatabase openDB(SqliteOpenOptions options) { - if (sqliteOptions.wasmSqlite3 == null) { + Future openDB(SqliteOpenOptions options) async { + if (sqliteOptions.wasmSqlite3Loader == null) { throw ArgumentError('WASM Sqlite3 implementation was not provided'); } - return sqliteOptions.wasmSqlite3!.open("/" + path); + final sqlite = await sqliteOptions.wasmSqlite3Loader!(); + return sqlite.open("/" + path); } @override List pragmaStatements(SqliteOpenOptions options) { - // WAL mode is not supported on web + // WAL mode is not supported return []; } } diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index d9fc318..0d43841 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,23 +1,26 @@ export './open_factory/abstract_open_factory.dart'; +import 'dart:async'; + import 'package:sqlite3/common.dart'; import 'package:sqlite_async/definitions.dart'; -import './open_factory/stub_sqlite_open_factory.dart' as base - if (dart.library.io) './open_factory/native/native_sqlite_open_factory.dart' - if (dart.library.html) './open_factory/web/web_sqlite_open_factory.dart'; +import './open_factory/open_factory_adapter.dart' as base; -class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { +class DefaultSqliteOpenFactory + extends AbstractDefaultSqliteOpenFactory { late AbstractDefaultSqliteOpenFactory adapter; DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}) { - adapter = base.DefaultSqliteOpenFactory(path: path, sqliteOptions: super.sqliteOptions) as AbstractDefaultSqliteOpenFactory; - } + adapter = base.DefaultSqliteOpenFactory( + path: path, sqliteOptions: super.sqliteOptions) + as AbstractDefaultSqliteOpenFactory; + } @override - T openDB(SqliteOpenOptions options) { + FutureOr openDB(SqliteOpenOptions options) { return adapter.openDB(options); } @@ -25,4 +28,4 @@ class DefaultSqliteOpenFactory extends AbstractDefault List pragmaStatements(SqliteOpenOptions options) { return adapter.pragmaStatements(options); } -} \ No newline at end of file +} diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index adb2a9f..a5d17b6 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -1,5 +1,20 @@ +import 'dart:async'; + import 'package:sqlite3/wasm.dart'; +Future loadWasmSqlite() async { + // TODO conditionally load debug version and specify DB name + final wasmSqlite3 = + await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); + + wasmSqlite3.registerVirtualFileSystem( + await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), + makeDefault: true, + ); + + return wasmSqlite3; +} + class SqliteOptions { /// SQLite journal mode. Defaults to [SqliteJournalMode.wal]. final SqliteJournalMode? journalMode; @@ -22,19 +37,19 @@ class SqliteOptions { /// makeDefault: true, /// ); /// Pass the initialized wasmSqlite3 here - final WasmSqlite3? wasmSqlite3; + final FutureOr Function()? wasmSqlite3Loader; const SqliteOptions.defaults() : journalMode = SqliteJournalMode.wal, journalSizeLimit = 6 * 1024 * 1024, // 1.5x the default checkpoint size synchronous = SqliteSynchronous.normal, - wasmSqlite3 = null; + wasmSqlite3Loader = loadWasmSqlite; const SqliteOptions( {this.journalMode = SqliteJournalMode.wal, this.journalSizeLimit = 6 * 1024 * 1024, this.synchronous = SqliteSynchronous.normal, - this.wasmSqlite3 = null}); + this.wasmSqlite3Loader = loadWasmSqlite}); } /// SQLite journal mode. Set on the primary connection. diff --git a/test/util.dart b/test/util.dart index 29662ed..79bd81b 100644 --- a/test/util.dart +++ b/test/util.dart @@ -1,3 +1,4 @@ +import 'dart:async'; import 'dart:ffi'; import 'dart:io'; import 'dart:isolate'; @@ -22,11 +23,11 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { this.sqlitePath = defaultSqlitePath}); @override - CommonDatabase open(SqliteOpenOptions options) { + FutureOr open(SqliteOpenOptions options) async { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { return DynamicLibrary.open(sqlitePath); }); - final db = super.open(options); + final db = await super.open(options); db.createFunction( functionName: 'test_sleep', From f433b234ccaaedd5aa32f0117fa4ff64cc1d5a99 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 12 Jan 2024 13:38:03 +0200 Subject: [PATCH 09/57] wip: added synchronous db implementation --- lib/src/database/web/web_db_context.dart | 59 +++++++++++++++++++ lib/src/database/web/web_sqlite_database.dart | 24 +++++--- pubspec.yaml | 1 + 3 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 lib/src/database/web/web_db_context.dart diff --git a/lib/src/database/web/web_db_context.dart b/lib/src/database/web/web_db_context.dart new file mode 100644 index 0000000..fd30a8b --- /dev/null +++ b/lib/src/database/web/web_db_context.dart @@ -0,0 +1,59 @@ +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; + +class WebReadContext implements SqliteReadContext { + CommonDatabase db; + + WebReadContext(CommonDatabase this.db); + + @override + Future computeWithDatabase( + Future Function(CommonDatabase db) compute) { + return compute(db); + } + + @override + Future get(String sql, [List parameters = const []]) async { + return db.select(sql, parameters).first; + } + + @override + Future getAll(String sql, + [List parameters = const []]) async { + return db.select(sql, parameters); + } + + @override + Future getOptional(String sql, + [List parameters = const []]) async { + try { + return db.select(sql, parameters).first; + } catch (ex) { + return null; + } + } +} + +class WebWriteContext extends WebReadContext implements SqliteWriteContext { + WebWriteContext(CommonDatabase super.db); + + @override + Future execute(String sql, + [List parameters = const []]) async { + final result = db.select(sql, parameters); + return result; + } + + @override + Future executeBatch( + String sql, List> parameterSets) async { + final statement = db.prepare(sql, checkNoTail: true); + try { + for (var parameters in parameterSets) { + statement.execute(parameters); + } + } finally { + statement.dispose(); + } + } +} diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database.dart index 3b36bf8..e5ed058 100644 --- a/lib/src/database/web/web_sqlite_database.dart +++ b/lib/src/database/web/web_sqlite_database.dart @@ -1,13 +1,16 @@ +import 'dart:async'; + import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/sqlite_connection.dart'; - -import '../abstract_sqlite_database.dart'; +import 'package:mutex/mutex.dart'; +import 'package:sqlite_async/src/database/web/web_db_context.dart'; class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); + late Mutex mutex; + late final CommonDatabase con; // late final Future _initialized; @@ -45,32 +48,37 @@ class SqliteDatabase extends AbstractSqliteDatabase { super.openFactory = openFactory; super.maxReaders = maxReaders; updates = updatesController.stream; + mutex = Mutex(); isInitialized = _init(); } Future _init() async { con = await openFactory .open(SqliteOpenOptions(primaryConnection: true, readOnly: false)); + con.updates.forEach((element) { + final tables = Set(); + tables.add(element.tableName); + updatesController.add(UpdateNotification(tables)); + }); } @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - throw UnimplementedError(); + return mutex.protect(() => callback(WebReadContext(con))); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - throw UnimplementedError(); + return mutex.protect(() => callback(WebWriteContext(con))); } @override - Future close() { - // TODO: implement close - throw UnimplementedError(); + Future close() async { + con.dispose(); } @override diff --git a/pubspec.yaml b/pubspec.yaml index 1c9d84f..eb60706 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -9,6 +9,7 @@ dependencies: sqlite3: '>=1.10.1 <3.0.0' async: ^2.10.0 collection: ^1.17.0 + mutex: ^3.1.0 dev_dependencies: lints: ^2.0.0 From 58453eef5c818b6f29428a9ac10843205abdbf4c Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 15 Jan 2024 16:30:51 +0200 Subject: [PATCH 10/57] bug fixes --- .../web/web_sqlite_open_factory.dart | 16 +++++++++++++--- lib/src/sqlite_database.dart | 7 ++++--- lib/src/sqlite_options.dart | 13 +++++++++---- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory.dart index 986c5e5..ddcc320 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory.dart @@ -4,9 +4,14 @@ import '../abstract_open_factory.dart'; class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { - const DefaultSqliteOpenFactory( + // TODO only 1 connection for now + CommonDatabase? connection; + + DefaultSqliteOpenFactory( {required super.path, - super.sqliteOptions = const SqliteOptions.defaults()}); + super.sqliteOptions = const SqliteOptions.defaults()}) { + connection = null; + } @override Future openDB(SqliteOpenOptions options) async { @@ -14,8 +19,13 @@ class DefaultSqliteOpenFactory throw ArgumentError('WASM Sqlite3 implementation was not provided'); } + // TODO, only 1 connection for now + if (connection != null) { + return connection!; + } + final sqlite = await sqliteOptions.wasmSqlite3Loader!(); - return sqlite.open("/" + path); + return connection = sqlite.open("/" + path); } @override diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 3a4823d..77a7850 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -1,7 +1,6 @@ // This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an // To conditionally export an implementation for either web or "native" platforms // The sqlite library uses dart:ffi which is not supported on web - import 'package:sqlite_async/sqlite_async.dart'; export 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; import './database/sqlite_database_adapter.dart' as base; @@ -29,9 +28,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = + super.openFactory = DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - adapter = base.SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + adapter = + base.SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); updates = adapter.updates; } @@ -47,6 +47,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { SqliteDatabase.withFactory(SqliteOpenFactory openFactory, {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { super.maxReaders = maxReaders; + super.openFactory = openFactory; adapter = base.SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); isInitialized = adapter.isInitialized; diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index a5d17b6..78c0524 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -2,17 +2,22 @@ import 'dart:async'; import 'package:sqlite3/wasm.dart'; +WasmSqlite3? _wasmSqlite = null; + Future loadWasmSqlite() async { + if (_wasmSqlite != null) { + return _wasmSqlite!; + } + // TODO conditionally load debug version and specify DB name - final wasmSqlite3 = - await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); + _wasmSqlite = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); - wasmSqlite3.registerVirtualFileSystem( + _wasmSqlite!.registerVirtualFileSystem( await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), makeDefault: true, ); - return wasmSqlite3; + return _wasmSqlite!; } class SqliteOptions { From 40fbd1df3f2f54f2d9d76018044e1f21cef77067 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 16 Jan 2024 10:12:24 +0200 Subject: [PATCH 11/57] sqlite version note --- README.md | 5 ++++- lib/src/open_factory/abstract_open_factory.dart | 2 ++ pubspec.yaml | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f3f8e6f..1f320c8 100644 --- a/README.md +++ b/README.md @@ -74,4 +74,7 @@ void main() async { await db.close(); } -``` \ No newline at end of file +``` + +# Web +Web requires sqlite3.dart version 2.3.0 or greater with the matching WASM file provided. \ No newline at end of file diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/open_factory/abstract_open_factory.dart index 2adec66..03c73e3 100644 --- a/lib/src/open_factory/abstract_open_factory.dart +++ b/lib/src/open_factory/abstract_open_factory.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'package:meta/meta.dart'; import 'package:sqlite3/common.dart' as sqlite; import '../../definitions.dart'; @@ -49,6 +50,7 @@ abstract class AbstractDefaultSqliteOpenFactory List pragmaStatements(SqliteOpenOptions options); + @protected FutureOr openDB(SqliteOpenOptions options); @override diff --git a/pubspec.yaml b/pubspec.yaml index eb60706..886d37a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,10 +3,10 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.5.2 repository: https://github.com/journeyapps/sqlite_async.dart environment: - sdk: '>=2.19.1 <4.0.0' + sdk: ">=2.19.1 <4.0.0" dependencies: - sqlite3: '>=1.10.1 <3.0.0' + sqlite3: ">=2.3.0 <3.0.0" async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From c7f22f88083efedb18949485cabfa139149e2279 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 16 Jan 2024 17:37:10 +0200 Subject: [PATCH 12/57] added async DB operations with Drift --- lib/src/database/web/web_db_context.dart | 26 ++--- lib/src/database/web/web_locks.dart | 12 +++ lib/src/database/web/web_sqlite_database.dart | 26 +++-- .../open_factory/abstract_open_factory.dart | 8 ++ .../native/native_sqlite_open_factory.dart | 8 ++ .../stub_sqlite_open_factory.dart | 9 ++ .../web/web_sqlite_open_factory.dart | 99 ++++++++++++++++--- lib/src/sqlite_connection.dart | 14 +++ lib/src/sqlite_open_factory.dart | 5 + lib/src/sqlite_options.dart | 41 +++----- pubspec.yaml | 1 + 11 files changed, 176 insertions(+), 73 deletions(-) create mode 100644 lib/src/database/web/web_locks.dart diff --git a/lib/src/database/web/web_db_context.dart b/lib/src/database/web/web_db_context.dart index fd30a8b..1067458 100644 --- a/lib/src/database/web/web_db_context.dart +++ b/lib/src/database/web/web_db_context.dart @@ -1,20 +1,22 @@ +import 'dart:async'; + import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; class WebReadContext implements SqliteReadContext { - CommonDatabase db; + SQLExecutor db; - WebReadContext(CommonDatabase this.db); + WebReadContext(SQLExecutor this.db); @override Future computeWithDatabase( Future Function(CommonDatabase db) compute) { - return compute(db); + throw UnimplementedError(); } @override Future get(String sql, [List parameters = const []]) async { - return db.select(sql, parameters).first; + return (await db.select(sql, parameters)).first; } @override @@ -27,7 +29,7 @@ class WebReadContext implements SqliteReadContext { Future getOptional(String sql, [List parameters = const []]) async { try { - return db.select(sql, parameters).first; + return (await db.select(sql, parameters)).first; } catch (ex) { return null; } @@ -35,25 +37,17 @@ class WebReadContext implements SqliteReadContext { } class WebWriteContext extends WebReadContext implements SqliteWriteContext { - WebWriteContext(CommonDatabase super.db); + WebWriteContext(SQLExecutor super.db); @override Future execute(String sql, [List parameters = const []]) async { - final result = db.select(sql, parameters); - return result; + return db.select(sql, parameters); } @override Future executeBatch( String sql, List> parameterSets) async { - final statement = db.prepare(sql, checkNoTail: true); - try { - for (var parameters in parameterSets) { - statement.execute(parameters); - } - } finally { - statement.dispose(); - } + return db.executeBatch(sql, parameterSets); } } diff --git a/lib/src/database/web/web_locks.dart b/lib/src/database/web/web_locks.dart new file mode 100644 index 0000000..7963820 --- /dev/null +++ b/lib/src/database/web/web_locks.dart @@ -0,0 +1,12 @@ +@JS() +library navigator_locks; + +import 'package:js/js.dart'; + +@JS('navigator.locks') +external NavigatorLocks navigatorLocks; + +abstract class NavigatorLocks { + Future request(String name, Function callbacks); + // Future request(String name, Future Function(dynamic lock) callback); +} diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database.dart index e5ed058..d4a13d4 100644 --- a/lib/src/database/web/web_sqlite_database.dart +++ b/lib/src/database/web/web_sqlite_database.dart @@ -1,6 +1,4 @@ import 'dart:async'; - -import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:mutex/mutex.dart'; import 'package:sqlite_async/src/database/web/web_db_context.dart'; @@ -9,9 +7,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); + late final Future executorFuture; late Mutex mutex; - - late final CommonDatabase con; + late final SQLExecutor executor; + late final String dbPath; // late final Future _initialized; @@ -45,19 +44,17 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 4. Creating temporary views or triggers. SqliteDatabase.withFactory(SqliteOpenFactory openFactory, {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { - super.openFactory = openFactory; - super.maxReaders = maxReaders; + executorFuture = openFactory.openWeb( + SqliteOpenOptions(primaryConnection: true, readOnly: false)) + as Future; updates = updatesController.stream; mutex = Mutex(); isInitialized = _init(); } Future _init() async { - con = await openFactory - .open(SqliteOpenOptions(primaryConnection: true, readOnly: false)); - con.updates.forEach((element) { - final tables = Set(); - tables.add(element.tableName); + executor = await executorFuture; + executor.updateStream.forEach((tables) { updatesController.add(UpdateNotification(tables)); }); } @@ -66,19 +63,20 @@ class SqliteDatabase extends AbstractSqliteDatabase { Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.protect(() => callback(WebReadContext(con))); + return mutex.protect(() => callback(WebReadContext(executor))); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.protect(() => callback(WebWriteContext(con))); + return mutex.protect(() => callback(WebWriteContext(executor))); } @override Future close() async { - con.dispose(); + await isInitialized; + await executor.close(); } @override diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/open_factory/abstract_open_factory.dart index 03c73e3..12585d6 100644 --- a/lib/src/open_factory/abstract_open_factory.dart +++ b/lib/src/open_factory/abstract_open_factory.dart @@ -9,7 +9,15 @@ import '../../definitions.dart'; /// Since connections are opened in dedicated background isolates, this class /// must be safe to pass to different isolates. abstract class SqliteOpenFactory { + String get path; + FutureOr open(SqliteOpenOptions options); + + /// This includes a limited set of async only SQL APIs required + /// for SqliteDatabase connections + FutureOr openWeb(SqliteOpenOptions options) { + throw UnimplementedError(); + } } class SqliteOpenOptions { diff --git a/lib/src/open_factory/native/native_sqlite_open_factory.dart b/lib/src/open_factory/native/native_sqlite_open_factory.dart index c93f3c6..e9c3ae1 100644 --- a/lib/src/open_factory/native/native_sqlite_open_factory.dart +++ b/lib/src/open_factory/native/native_sqlite_open_factory.dart @@ -1,4 +1,7 @@ +import 'dart:async'; + import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import '../abstract_open_factory.dart'; @@ -35,4 +38,9 @@ class DefaultSqliteOpenFactory } return statements; } + + @override + FutureOr openWeb(SqliteOpenOptions options) { + throw UnimplementedError(); + } } diff --git a/lib/src/open_factory/stub_sqlite_open_factory.dart b/lib/src/open_factory/stub_sqlite_open_factory.dart index 1dcec61..f7d77a2 100644 --- a/lib/src/open_factory/stub_sqlite_open_factory.dart +++ b/lib/src/open_factory/stub_sqlite_open_factory.dart @@ -1,4 +1,7 @@ +import 'dart:async'; + import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; @@ -16,4 +19,10 @@ class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { List pragmaStatements(SqliteOpenOptions options) { throw UnimplementedError(); } + + @override + FutureOr openWeb(SqliteOpenOptions options) { + // TODO: implement openWeb + throw UnimplementedError(); + } } diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory.dart index ddcc320..5e0e2ac 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory.dart @@ -1,36 +1,109 @@ +import 'dart:async'; + +import 'package:drift/drift.dart'; +import 'package:drift/wasm.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite3/wasm.dart'; import '../abstract_open_factory.dart'; +class DriftWebSQLExecutor extends SQLExecutor { + WasmDatabaseResult db; + final StreamController> updatesController = + StreamController.broadcast(); + + DriftWebSQLExecutor(WasmDatabaseResult this.db) { + // Pass on table updates + db.resolvedExecutor.streamQueries + .updatesForSync(TableUpdateQuery.any()) + .forEach((tables) { + updatesController.add(tables.map((e) => e.table).toSet()); + }); + updateStream = updatesController.stream; + } + + @override + close() { + return db.resolvedExecutor.close(); + } + + @override + Future executeBatch(String sql, List> parameterSets) { + return db.resolvedExecutor.runBatched(BatchedStatements( + [sql], [ArgumentsForBatchedStatement(0, parameterSets)])); + } + + @override + FutureOr select(String sql, + [List parameters = const []]) async { + final result = await db.resolvedExecutor.runSelect(sql, parameters); + if (result.isEmpty) { + return ResultSet([], [], []); + } + return ResultSet(result.first.keys.toList(), [], + result.map((e) => e.values.toList()).toList()); + } +} + +class SqliteUser extends QueryExecutorUser { + @override + Future beforeOpen( + QueryExecutor executor, OpeningDetails details) async {} + + @override + int get schemaVersion => 1; +} + class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { // TODO only 1 connection for now - CommonDatabase? connection; + SQLExecutor? executor; DefaultSqliteOpenFactory( {required super.path, - super.sqliteOptions = const SqliteOptions.defaults()}) { - connection = null; - } + super.sqliteOptions = const SqliteOptions.defaults()}) {} @override + + /// It is possible to open a CommonDatabase in the main Dart/JS context with standard sqlite3.dart, + /// This connection requires an external Webworker implementation for asynchronous operations. Future openDB(SqliteOpenOptions options) async { - if (sqliteOptions.wasmSqlite3Loader == null) { - throw ArgumentError('WASM Sqlite3 implementation was not provided'); - } + final wasmSqlite = await WasmSqlite3.loadFromUrl( + Uri.parse(sqliteOptions.webSqliteOptions.wasmUri)); - // TODO, only 1 connection for now - if (connection != null) { - return connection!; + wasmSqlite.registerVirtualFileSystem( + await IndexedDbFileSystem.open(dbName: path), + makeDefault: true, + ); + + return wasmSqlite.open(path); + } + + @override + + /// The Drift SQLite package provides built in async Webworker functionality + /// and automatic persistence storage selection. + /// Due to being asynchronous, the underlaying CommonDatabase is not accessible + Future openWeb(SqliteOpenOptions options) async { + if (executor != null) { + return executor!; } - final sqlite = await sqliteOptions.wasmSqlite3Loader!(); - return connection = sqlite.open("/" + path); + final db = await WasmDatabase.open( + databaseName: path, + sqlite3Uri: Uri.parse(sqliteOptions.webSqliteOptions.wasmUri), + driftWorkerUri: Uri.parse(sqliteOptions.webSqliteOptions.workerUri), + ); + + await db.resolvedExecutor.ensureOpen(SqliteUser()); + executor = DriftWebSQLExecutor(db); + + return executor!; } @override List pragmaStatements(SqliteOpenOptions options) { - // WAL mode is not supported + // WAL mode is not supported on Web return []; } } diff --git a/lib/src/sqlite_connection.dart b/lib/src/sqlite_connection.dart index 6df5ba8..bca675c 100644 --- a/lib/src/sqlite_connection.dart +++ b/lib/src/sqlite_connection.dart @@ -1,5 +1,19 @@ +import 'dart:async'; + import 'package:sqlite3/common.dart' as sqlite; +// Abstract class which provides base methods required for Context providers +abstract class SQLExecutor { + Stream> updateStream = Stream.empty(); + + FutureOr select(String sql, + [List parameters = const []]); + + FutureOr executeBatch(String sql, List> parameterSets) {} + + Future close(); +} + /// Abstract class representing calls available in a read-only or read-write context. abstract class SqliteReadContext { /// Execute a read-only (SELECT) query and return the results. diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 0d43841..cd6bb70 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -28,4 +28,9 @@ class DefaultSqliteOpenFactory List pragmaStatements(SqliteOpenOptions options) { return adapter.pragmaStatements(options); } + + @override + FutureOr openWeb(SqliteOpenOptions options) { + return adapter.openWeb(options); + } } diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index 78c0524..3bf4f15 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -1,23 +1,13 @@ -import 'dart:async'; +class WebSqliteOptions { + final String workerUri; + final String wasmUri; -import 'package:sqlite3/wasm.dart'; + const WebSqliteOptions.defaults() + : workerUri = 'drift_worker.js', + wasmUri = 'sqlite3.wasm'; -WasmSqlite3? _wasmSqlite = null; - -Future loadWasmSqlite() async { - if (_wasmSqlite != null) { - return _wasmSqlite!; - } - - // TODO conditionally load debug version and specify DB name - _wasmSqlite = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); - - _wasmSqlite!.registerVirtualFileSystem( - await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), - makeDefault: true, - ); - - return _wasmSqlite!; + const WebSqliteOptions( + {this.wasmUri = 'sqlite3.wasm', this.workerUri = 'drift_worker.js'}); } class SqliteOptions { @@ -33,28 +23,19 @@ class SqliteOptions { /// attempt to truncate the file afterwards. final int? journalSizeLimit; - /// The implementation for SQLite - /// This is required for Web WASM - /// final wasmSqlite3 = - /// await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.debug.wasm')); - /// wasmSqlite3.registerVirtualFileSystem( - /// await IndexedDbFileSystem.open(dbName: 'sqlite3-example'), - /// makeDefault: true, - /// ); - /// Pass the initialized wasmSqlite3 here - final FutureOr Function()? wasmSqlite3Loader; + final WebSqliteOptions webSqliteOptions; const SqliteOptions.defaults() : journalMode = SqliteJournalMode.wal, journalSizeLimit = 6 * 1024 * 1024, // 1.5x the default checkpoint size synchronous = SqliteSynchronous.normal, - wasmSqlite3Loader = loadWasmSqlite; + webSqliteOptions = const WebSqliteOptions.defaults(); const SqliteOptions( {this.journalMode = SqliteJournalMode.wal, this.journalSizeLimit = 6 * 1024 * 1024, this.synchronous = SqliteSynchronous.normal, - this.wasmSqlite3Loader = loadWasmSqlite}); + this.webSqliteOptions = const WebSqliteOptions.defaults()}); } /// SQLite journal mode. Set on the primary connection. diff --git a/pubspec.yaml b/pubspec.yaml index 886d37a..5a40246 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,6 +6,7 @@ environment: sdk: ">=2.19.1 <4.0.0" dependencies: + drift: ^2.14.1 sqlite3: ">=2.3.0 <3.0.0" async: ^2.10.0 collection: ^1.17.0 From 7e406a3a425c8e9faf886a850e696c0334eb75e0 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 08:53:27 +0200 Subject: [PATCH 13/57] wip: drift --- lib/drift.dart | 5 +++++ .../open_factory/web/web_sqlite_open_factory.dart | 15 ++++++--------- pubspec.yaml | 3 ++- 3 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 lib/drift.dart diff --git a/lib/drift.dart b/lib/drift.dart new file mode 100644 index 0000000..e4cb41f --- /dev/null +++ b/lib/drift.dart @@ -0,0 +1,5 @@ +/// Re-exports [Drift](https://pub.dev/packages/drift) to expose drift without +/// adding it as a direct dependency. +library; + +export 'package:drift/wasm.dart'; diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory.dart index 5e0e2ac..25b27d6 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory.dart @@ -9,17 +9,14 @@ import '../abstract_open_factory.dart'; class DriftWebSQLExecutor extends SQLExecutor { WasmDatabaseResult db; - final StreamController> updatesController = - StreamController.broadcast(); DriftWebSQLExecutor(WasmDatabaseResult this.db) { // Pass on table updates - db.resolvedExecutor.streamQueries + updateStream = db.resolvedExecutor.streamQueries .updatesForSync(TableUpdateQuery.any()) - .forEach((tables) { - updatesController.add(tables.map((e) => e.table).toSet()); + .map((tables) { + return tables.map((e) => e.table).toSet(); }); - updateStream = updatesController.stream; } @override @@ -29,8 +26,8 @@ class DriftWebSQLExecutor extends SQLExecutor { @override Future executeBatch(String sql, List> parameterSets) { - return db.resolvedExecutor.runBatched(BatchedStatements( - [sql], [ArgumentsForBatchedStatement(0, parameterSets)])); + return db.resolvedExecutor.runBatched(BatchedStatements([sql], + parameterSets.map((e) => ArgumentsForBatchedStatement(0, e)).toList())); } @override @@ -95,8 +92,8 @@ class DefaultSqliteOpenFactory driftWorkerUri: Uri.parse(sqliteOptions.webSqliteOptions.workerUri), ); - await db.resolvedExecutor.ensureOpen(SqliteUser()); executor = DriftWebSQLExecutor(db); + await db.resolvedExecutor.ensureOpen(SqliteUser()); return executor!; } diff --git a/pubspec.yaml b/pubspec.yaml index 5a40246..7576338 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,7 +6,8 @@ environment: sdk: ">=2.19.1 <4.0.0" dependencies: - drift: ^2.14.1 + drift: + path: "/Users/stevenontong/Documents/platform_code/powersync/drift/drift" sqlite3: ">=2.3.0 <3.0.0" async: ^2.10.0 collection: ^1.17.0 From c6f678ac23cdf77ebc24982a34f397eff0b13464 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 08:59:21 +0200 Subject: [PATCH 14/57] allow multiple web connections (leveraging web worker) --- lib/src/open_factory/web/web_sqlite_open_factory.dart | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory.dart index 25b27d6..f300663 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory.dart @@ -53,9 +53,6 @@ class SqliteUser extends QueryExecutorUser { class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { - // TODO only 1 connection for now - SQLExecutor? executor; - DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}) {} @@ -82,20 +79,16 @@ class DefaultSqliteOpenFactory /// and automatic persistence storage selection. /// Due to being asynchronous, the underlaying CommonDatabase is not accessible Future openWeb(SqliteOpenOptions options) async { - if (executor != null) { - return executor!; - } - final db = await WasmDatabase.open( databaseName: path, sqlite3Uri: Uri.parse(sqliteOptions.webSqliteOptions.wasmUri), driftWorkerUri: Uri.parse(sqliteOptions.webSqliteOptions.workerUri), ); - executor = DriftWebSQLExecutor(db); + final executor = DriftWebSQLExecutor(db); await db.resolvedExecutor.ensureOpen(SqliteUser()); - return executor!; + return executor; } @override From 52466582c0ca0030144320a94c4d1edec8ddb6f6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 13:18:59 +0200 Subject: [PATCH 15/57] wip --- .../database/abstract_sqlite_database.dart | 48 +++---------------- lib/src/database/native/connection_pool.dart | 2 +- ....dart => native_sqlite_database_impl.dart} | 30 +++++++----- .../native/sqlite_connection_impl.dart | 7 +-- lib/src/database/sqlite_database_adapter.dart | 5 -- lib/src/database/sqlite_database_impl.dart | 5 ++ lib/src/database/stub_sqlite_database.dart | 20 ++++++-- ...ase.dart => web_sqlite_database_impl.dart} | 43 +++++++++++------ .../abstract_isolate_connection_factory.dart | 2 +- .../native_isolate_connection_factory.dart | 16 +++---- .../stub_isolate_connection_factory.dart | 9 ++-- .../web/web_isolate_connection_factory.dart | 12 ++--- .../open_factory/abstract_open_factory.dart | 16 ++++--- ...t => native_sqlite_open_factory_impl.dart} | 10 ++-- .../open_factory/open_factory_adapter.dart | 5 -- lib/src/open_factory/open_factory_impl.dart | 5 ++ .../stub_sqlite_open_factory.dart | 8 ++-- ...dart => web_sqlite_open_factory_impl.dart} | 20 ++++---- lib/src/sqlite_connection.dart | 6 ++- lib/src/sqlite_database.dart | 40 ++++++++++------ lib/src/sqlite_open_factory.dart | 19 ++++---- pubspec.yaml | 1 + 22 files changed, 173 insertions(+), 156 deletions(-) rename lib/src/database/native/{native_sqlite_database.dart => native_sqlite_database_impl.dart} (89%) delete mode 100644 lib/src/database/sqlite_database_adapter.dart create mode 100644 lib/src/database/sqlite_database_impl.dart rename lib/src/database/web/{web_sqlite_database.dart => web_sqlite_database_impl.dart} (74%) rename lib/src/open_factory/native/{native_sqlite_open_factory.dart => native_sqlite_open_factory_impl.dart} (80%) delete mode 100644 lib/src/open_factory/open_factory_adapter.dart create mode 100644 lib/src/open_factory/open_factory_impl.dart rename lib/src/open_factory/web/{web_sqlite_open_factory.dart => web_sqlite_open_factory_impl.dart} (82%) diff --git a/lib/src/database/abstract_sqlite_database.dart b/lib/src/database/abstract_sqlite_database.dart index 01e247d..17c84ef 100644 --- a/lib/src/database/abstract_sqlite_database.dart +++ b/lib/src/database/abstract_sqlite_database.dart @@ -8,30 +8,28 @@ import '../../definitions.dart'; /// /// Use one instance per database file. If multiple instances are used, update /// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. -abstract class AbstractSqliteDatabase - with SqliteQueries - implements SqliteConnection { +abstract class AbstractSqliteDatabase extends SqliteConnection + with SqliteQueries { /// The maximum number of concurrent read transactions if not explicitly specified. static const int defaultMaxReaders = 5; /// Maximum number of concurrent read transactions. - late final int maxReaders; + int get maxReaders; /// Factory that opens a raw database connection in each isolate. /// /// This must be safe to pass to different isolates. /// /// Use a custom class for this to customize the open process. - late final SqliteOpenFactory openFactory; + SqliteOpenFactory get openFactory; /// Use this stream to subscribe to notifications of updates to tables. - @override - late final Stream updates; + Stream get updates; final StreamController updatesController = StreamController.broadcast(); - late final Future isInitialized; + Future get isInitialized; /// Wait for initialization to complete. /// @@ -44,38 +42,4 @@ abstract class AbstractSqliteDatabase /// /// Use this to access the database in background isolates. AbstractIsolateConnectionFactory isolateConnectionFactory(); - - /// Open a read-only transaction. - /// - /// Up to [maxReaders] read transactions can run concurrently. - /// After that, read transactions are queued. - /// - /// Read transactions can run concurrently to a write transaction. - /// - /// Changes from any write transaction are not visible to read transactions - /// started before it. - @override - Future readTransaction( - Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout}); - - /// Open a read-write transaction. - /// - /// Only a single write transaction can run at a time - any concurrent - /// transactions are queued. - /// - /// The write transaction is automatically committed when the callback finishes, - /// or rolled back on any error. - @override - Future writeTransaction( - Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout}); - - @override - Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}); - - @override - Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}); } diff --git a/lib/src/database/native/connection_pool.dart b/lib/src/database/native/connection_pool.dart index 80252ab..482952b 100644 --- a/lib/src/database/native/connection_pool.dart +++ b/lib/src/database/native/connection_pool.dart @@ -16,7 +16,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final List _readConnections = []; - final SqliteOpenFactory _factory; + final SqliteOpenFactory _factory; final SerializedPortClient _upstreamPort; @override diff --git a/lib/src/database/native/native_sqlite_database.dart b/lib/src/database/native/native_sqlite_database_impl.dart similarity index 89% rename from lib/src/database/native/native_sqlite_database.dart rename to lib/src/database/native/native_sqlite_database_impl.dart index 09cf844..d8e26f0 100644 --- a/lib/src/database/native/native_sqlite_database.dart +++ b/lib/src/database/native/native_sqlite_database_impl.dart @@ -1,14 +1,12 @@ import 'dart:async'; import 'dart:isolate'; -import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory_impl.dart'; import '../../../mutex.dart'; import '../../utils/database_utils.dart'; import '../../sqlite_connection.dart'; -import '../../open_factory/native/native_sqlite_open_factory.dart'; import '../../isolate_connection_factory/native/native_isolate_connection_factory.dart'; -import '../../open_factory/abstract_open_factory.dart'; import '../../sqlite_options.dart'; import '../../update_notification.dart'; import '../abstract_sqlite_database.dart'; @@ -20,9 +18,18 @@ import 'sqlite_connection_impl.dart'; /// /// Use one instance per database file. If multiple instances are used, update /// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. -class SqliteDatabase extends AbstractSqliteDatabase { +class SqliteDatabaseImplementation extends AbstractSqliteDatabase { @override - final SqliteOpenFactory openFactory; + final DefaultSqliteOpenFactoryImplementation openFactory; + + @override + late Stream updates; + + @override + int maxReaders; + + @override + late Future isInitialized; late final PortServer _eventsPort; @@ -42,13 +49,14 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// from the last committed write transaction. /// /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabase( + factory SqliteDatabaseImplementation( {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = - DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + final factory = DefaultSqliteOpenFactoryImplementation( + path: path, sqliteOptions: options); + return SqliteDatabaseImplementation.withFactory(factory, + maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -60,8 +68,8 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(this.openFactory, - {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + SqliteDatabaseImplementation.withFactory(this.openFactory, + {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { updates = updatesController.stream; _listenForEvents(); diff --git a/lib/src/database/native/sqlite_connection_impl.dart b/lib/src/database/native/sqlite_connection_impl.dart index b93cd6b..4ca30e8 100644 --- a/lib/src/database/native/sqlite_connection_impl.dart +++ b/lib/src/database/native/sqlite_connection_impl.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory_impl.dart'; import '../../utils/database_utils.dart'; import '../../mutex.dart'; @@ -29,7 +30,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { final bool readOnly; SqliteConnectionImpl( - {required SqliteOpenFactory openFactory, + {required openFactory, required Mutex mutex, required SerializedPortClient upstreamPort, this.updates, @@ -49,7 +50,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { return _isolateClient.closed; } - Future _open(SqliteOpenFactory openFactory, + Future _open(DefaultSqliteOpenFactoryImplementation openFactory, {required bool primary, required SerializedPortClient upstreamPort}) async { await _connectionMutex.lock(() async { @@ -334,7 +335,7 @@ class _SqliteConnectionParams { final SerializedPortClient port; final bool primary; - final SqliteOpenFactory openFactory; + final DefaultSqliteOpenFactoryImplementation openFactory; _SqliteConnectionParams( {required this.openFactory, diff --git a/lib/src/database/sqlite_database_adapter.dart b/lib/src/database/sqlite_database_adapter.dart deleted file mode 100644 index 9336dbf..0000000 --- a/lib/src/database/sqlite_database_adapter.dart +++ /dev/null @@ -1,5 +0,0 @@ -export 'stub_sqlite_database.dart' - // ignore: uri_does_not_exist - if (dart.library.io) './native/native_sqlite_database.dart' - // ignore: uri_does_not_exist - if (dart.library.html) './web/web_sqlite_database.dart'; diff --git a/lib/src/database/sqlite_database_impl.dart b/lib/src/database/sqlite_database_impl.dart new file mode 100644 index 0000000..108f3c2 --- /dev/null +++ b/lib/src/database/sqlite_database_impl.dart @@ -0,0 +1,5 @@ +export 'stub_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './native/native_sqlite_database_impl.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './web/web_sqlite_database_impl.dart'; diff --git a/lib/src/database/stub_sqlite_database.dart b/lib/src/database/stub_sqlite_database.dart index aa4a8c6..a1f3bf3 100644 --- a/lib/src/database/stub_sqlite_database.dart +++ b/lib/src/database/stub_sqlite_database.dart @@ -1,21 +1,33 @@ import 'package:sqlite_async/sqlite_async.dart'; -class SqliteDatabase extends AbstractSqliteDatabase { +class SqliteDatabaseImplementation extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); - factory SqliteDatabase( + @override + SqliteOpenFactory openFactory; + + @override + int maxReaders; + + factory SqliteDatabaseImplementation( {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { throw UnimplementedError(); } - SqliteDatabase.withFactory(SqliteOpenFactory openFactory, - {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + SqliteDatabaseImplementation.withFactory(this.openFactory, + {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { throw UnimplementedError(); } + @override + Future get isInitialized => throw UnimplementedError(); + + @override + Stream get updates => throw UnimplementedError(); + @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) { diff --git a/lib/src/database/web/web_sqlite_database.dart b/lib/src/database/web/web_sqlite_database_impl.dart similarity index 74% rename from lib/src/database/web/web_sqlite_database.dart rename to lib/src/database/web/web_sqlite_database_impl.dart index d4a13d4..7b1d4f3 100644 --- a/lib/src/database/web/web_sqlite_database.dart +++ b/lib/src/database/web/web_sqlite_database_impl.dart @@ -1,18 +1,32 @@ import 'dart:async'; +import 'package:meta/meta.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:mutex/mutex.dart'; import 'package:sqlite_async/src/database/web/web_db_context.dart'; -class SqliteDatabase extends AbstractSqliteDatabase { +class SqliteDatabaseImplementation extends AbstractSqliteDatabase { @override - bool get closed => throw UnimplementedError(); + bool get closed { + return executor == null || executor!.closed; + } + + @override + late Stream updates; + + @override + int maxReaders; + + @override + late Future isInitialized; + + @override + SqliteOpenFactory openFactory; late final Future executorFuture; late Mutex mutex; - late final SQLExecutor executor; - late final String dbPath; - // late final Future _initialized; + @protected + late SQLExecutor? executor; /// Open a SqliteDatabase. /// @@ -24,13 +38,14 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// from the last committed write transaction. /// /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabase( + factory SqliteDatabaseImplementation( {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { final factory = DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + return SqliteDatabaseImplementation.withFactory(factory, + maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -42,9 +57,9 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(SqliteOpenFactory openFactory, - {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { - executorFuture = openFactory.openWeb( + SqliteDatabaseImplementation.withFactory(this.openFactory, + {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + executorFuture = openFactory.openExecutor( SqliteOpenOptions(primaryConnection: true, readOnly: false)) as Future; updates = updatesController.stream; @@ -54,7 +69,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { Future _init() async { executor = await executorFuture; - executor.updateStream.forEach((tables) { + executor!.updateStream.forEach((tables) { updatesController.add(UpdateNotification(tables)); }); } @@ -63,20 +78,20 @@ class SqliteDatabase extends AbstractSqliteDatabase { Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.protect(() => callback(WebReadContext(executor))); + return mutex.protect(() => callback(WebReadContext(executor!))); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.protect(() => callback(WebWriteContext(executor))); + return mutex.protect(() => callback(WebWriteContext(executor!))); } @override Future close() async { await isInitialized; - await executor.close(); + await executor!.close(); } @override diff --git a/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart index 7f3febb..ee40f7f 100644 --- a/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart @@ -4,7 +4,7 @@ import 'package:sqlite_async/definitions.dart'; /// A connection factory that can be passed to different isolates. abstract class AbstractIsolateConnectionFactory { - late SqliteOpenFactory openFactory; + AbstractDefaultSqliteOpenFactory get openFactory; /// Open a new SqliteConnection. /// diff --git a/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart index 282553d..67b5aa1 100644 --- a/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart @@ -1,11 +1,9 @@ import 'dart:async'; import 'dart:isolate'; -import 'package:sqlite3/sqlite3.dart' as sqlite; - +import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory_impl.dart'; import '../../mutex.dart'; import '../../sqlite_connection.dart'; -import '../../sqlite_open_factory.dart'; import '../../update_notification.dart'; import '../../utils/native_database_utils.dart'; import '../../database/native/port_channel.dart'; @@ -14,15 +12,16 @@ import '../abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + @override + DefaultSqliteOpenFactoryImplementation openFactory; + SerializedMutex mutex; SerializedPortClient upstreamPort; IsolateConnectionFactory( - {required SqliteOpenFactory openFactory, + {required this.openFactory, required this.mutex, - required this.upstreamPort}) { - super.openFactory = openFactory; - } + required this.upstreamPort}); /// Open a new SqliteConnection. /// @@ -33,8 +32,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { var openMutex = mutex.open(); return _IsolateSqliteConnection( - openFactory: - openFactory as AbstractDefaultSqliteOpenFactory, + openFactory: openFactory, mutex: openMutex, upstreamPort: upstreamPort, readOnly: readOnly, diff --git a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart index 24d361c..68d5d88 100644 --- a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart @@ -6,11 +6,12 @@ import 'abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + @override + AbstractDefaultSqliteOpenFactory openFactory; + IsolateConnectionFactory({ - required SqliteOpenFactory openFactory, - }) { - super.openFactory = openFactory; - } + required this.openFactory, + }); /// Open a new SqliteConnection. /// diff --git a/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart index dc615f5..db47924 100644 --- a/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart @@ -8,11 +8,12 @@ import '../abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + @override + AbstractDefaultSqliteOpenFactory openFactory; + IsolateConnectionFactory({ - required SqliteOpenFactory openFactory, - }) { - super.openFactory = openFactory; - } + required this.openFactory, + }); /// Open a new SqliteConnection. /// @@ -30,8 +31,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { /// 2. Other connections are not notified of any updates to tables made within /// this connection. Future openRawDatabase({bool readOnly = false}) async { - final db = await openFactory + return openFactory .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); - return db; } } diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/open_factory/abstract_open_factory.dart index 12585d6..5981f4f 100644 --- a/lib/src/open_factory/abstract_open_factory.dart +++ b/lib/src/open_factory/abstract_open_factory.dart @@ -8,14 +8,15 @@ import '../../definitions.dart'; /// /// Since connections are opened in dedicated background isolates, this class /// must be safe to pass to different isolates. -abstract class SqliteOpenFactory { +abstract class SqliteOpenFactory { String get path; - FutureOr open(SqliteOpenOptions options); + FutureOr open(SqliteOpenOptions options); /// This includes a limited set of async only SQL APIs required /// for SqliteDatabase connections - FutureOr openWeb(SqliteOpenOptions options) { + FutureOr openExecutor(SqliteOpenOptions options) { throw UnimplementedError(); } } @@ -47,8 +48,9 @@ class SqliteOpenOptions { /// to configure the connection. /// /// Override the [open] method to customize the process. -abstract class AbstractDefaultSqliteOpenFactory - implements SqliteOpenFactory { +abstract class AbstractDefaultSqliteOpenFactory< + Database extends sqlite.CommonDatabase, Executor extends SQLExecutor> + implements SqliteOpenFactory { final String path; final SqliteOptions sqliteOptions; @@ -59,10 +61,10 @@ abstract class AbstractDefaultSqliteOpenFactory List pragmaStatements(SqliteOpenOptions options); @protected - FutureOr openDB(SqliteOpenOptions options); + FutureOr openDB(SqliteOpenOptions options); @override - FutureOr open(SqliteOpenOptions options) async { + FutureOr open(SqliteOpenOptions options) async { var db = await openDB(options); for (var statement in pragmaStatements(options)) { diff --git a/lib/src/open_factory/native/native_sqlite_open_factory.dart b/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart similarity index 80% rename from lib/src/open_factory/native/native_sqlite_open_factory.dart rename to lib/src/open_factory/native/native_sqlite_open_factory_impl.dart index e9c3ae1..116e006 100644 --- a/lib/src/open_factory/native/native_sqlite_open_factory.dart +++ b/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart @@ -1,13 +1,13 @@ import 'dart:async'; import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_async/src/open_factory/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; -import '../abstract_open_factory.dart'; -class DefaultSqliteOpenFactory - extends AbstractDefaultSqliteOpenFactory { - const DefaultSqliteOpenFactory( +class DefaultSqliteOpenFactoryImplementation + extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactoryImplementation( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); @@ -40,7 +40,7 @@ class DefaultSqliteOpenFactory } @override - FutureOr openWeb(SqliteOpenOptions options) { + FutureOr openExecutor(SqliteOpenOptions options) { throw UnimplementedError(); } } diff --git a/lib/src/open_factory/open_factory_adapter.dart b/lib/src/open_factory/open_factory_adapter.dart deleted file mode 100644 index af03ec2..0000000 --- a/lib/src/open_factory/open_factory_adapter.dart +++ /dev/null @@ -1,5 +0,0 @@ -export 'stub_sqlite_open_factory.dart' - // ignore: uri_does_not_exist - if (dart.library.io) './native/native_sqlite_open_factory.dart' - // ignore: uri_does_not_exist - if (dart.library.html) './web/web_sqlite_open_factory.dart'; diff --git a/lib/src/open_factory/open_factory_impl.dart b/lib/src/open_factory/open_factory_impl.dart new file mode 100644 index 0000000..9504484 --- /dev/null +++ b/lib/src/open_factory/open_factory_impl.dart @@ -0,0 +1,5 @@ +export 'stub_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) './native/native_sqlite_open_factory_impl.dart' + // ignore: uri_does_not_exist + if (dart.library.html) './web/web_sqlite_open_factory_impl.dart'; diff --git a/lib/src/open_factory/stub_sqlite_open_factory.dart b/lib/src/open_factory/stub_sqlite_open_factory.dart index f7d77a2..f6047fc 100644 --- a/lib/src/open_factory/stub_sqlite_open_factory.dart +++ b/lib/src/open_factory/stub_sqlite_open_factory.dart @@ -5,8 +5,9 @@ import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; -class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { - const DefaultSqliteOpenFactory( +class DefaultSqliteOpenFactoryImplementation + extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactoryImplementation( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); @@ -21,8 +22,7 @@ class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { } @override - FutureOr openWeb(SqliteOpenOptions options) { - // TODO: implement openWeb + FutureOr openExecutor(SqliteOpenOptions options) { throw UnimplementedError(); } } diff --git a/lib/src/open_factory/web/web_sqlite_open_factory.dart b/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart similarity index 82% rename from lib/src/open_factory/web/web_sqlite_open_factory.dart rename to lib/src/open_factory/web/web_sqlite_open_factory_impl.dart index f300663..0f32739 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart @@ -10,7 +10,10 @@ import '../abstract_open_factory.dart'; class DriftWebSQLExecutor extends SQLExecutor { WasmDatabaseResult db; - DriftWebSQLExecutor(WasmDatabaseResult this.db) { + @override + bool closed = false; + + DriftWebSQLExecutor(this.db) { // Pass on table updates updateStream = db.resolvedExecutor.streamQueries .updatesForSync(TableUpdateQuery.any()) @@ -21,6 +24,7 @@ class DriftWebSQLExecutor extends SQLExecutor { @override close() { + closed = true; return db.resolvedExecutor.close(); } @@ -51,11 +55,11 @@ class SqliteUser extends QueryExecutorUser { int get schemaVersion => 1; } -class DefaultSqliteOpenFactory - extends AbstractDefaultSqliteOpenFactory { - DefaultSqliteOpenFactory( +class DefaultSqliteOpenFactoryImplementation + extends AbstractDefaultSqliteOpenFactory { + DefaultSqliteOpenFactoryImplementation( {required super.path, - super.sqliteOptions = const SqliteOptions.defaults()}) {} + super.sqliteOptions = const SqliteOptions.defaults()}); @override @@ -75,10 +79,10 @@ class DefaultSqliteOpenFactory @override - /// The Drift SQLite package provides built in async Webworker functionality + /// The Drift SQLite package provides built in async Web worker functionality /// and automatic persistence storage selection. - /// Due to being asynchronous, the underlaying CommonDatabase is not accessible - Future openWeb(SqliteOpenOptions options) async { + /// Due to being asynchronous, the under laying CommonDatabase is not accessible + Future openExecutor(SqliteOpenOptions options) async { final db = await WasmDatabase.open( databaseName: path, sqlite3Uri: Uri.parse(sqliteOptions.webSqliteOptions.wasmUri), diff --git a/lib/src/sqlite_connection.dart b/lib/src/sqlite_connection.dart index bca675c..6a959c7 100644 --- a/lib/src/sqlite_connection.dart +++ b/lib/src/sqlite_connection.dart @@ -4,14 +4,16 @@ import 'package:sqlite3/common.dart' as sqlite; // Abstract class which provides base methods required for Context providers abstract class SQLExecutor { + bool get closed; + Stream> updateStream = Stream.empty(); + Future close(); + FutureOr select(String sql, [List parameters = const []]); FutureOr executeBatch(String sql, List> parameterSets) {} - - Future close(); } /// Abstract class representing calls available in a read-only or read-write context. diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 77a7850..40bc032 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -3,16 +3,33 @@ // The sqlite library uses dart:ffi which is not supported on web import 'package:sqlite_async/sqlite_async.dart'; export 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; -import './database/sqlite_database_adapter.dart' as base; +import 'database/sqlite_database_impl.dart'; class SqliteDatabase extends AbstractSqliteDatabase { static const int defaultMaxReaders = AbstractSqliteDatabase.defaultMaxReaders; - /// Use this stream to subscribe to notifications of updates to tables. + late AbstractSqliteDatabase adapter; + @override - late final Stream updates; + int get maxReaders { + return adapter.maxReaders; + } - late AbstractSqliteDatabase adapter; + @override + Future get isInitialized { + return adapter.isInitialized; + } + + @override + SqliteOpenFactory get openFactory { + return adapter.openFactory; + } + + /// Use this stream to subscribe to notifications of updates to tables. + @override + Stream get updates { + return adapter.updates; + } /// Open a SqliteDatabase. /// @@ -28,11 +45,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { - super.openFactory = + final openFactory = DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - adapter = - base.SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); - updates = adapter.updates; + adapter = SqliteDatabaseImplementation.withFactory(openFactory, + maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -46,12 +62,8 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 4. Creating temporary views or triggers. SqliteDatabase.withFactory(SqliteOpenFactory openFactory, {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { - super.maxReaders = maxReaders; - super.openFactory = openFactory; - adapter = - base.SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); - isInitialized = adapter.isInitialized; - updates = adapter.updates; + adapter = SqliteDatabaseImplementation.withFactory(openFactory, + maxReaders: maxReaders); } @override diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index cd6bb70..73167b9 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,26 +1,23 @@ export './open_factory/abstract_open_factory.dart'; import 'dart:async'; - import 'package:sqlite3/common.dart'; import 'package:sqlite_async/definitions.dart'; -import './open_factory/open_factory_adapter.dart' as base; +import 'open_factory/open_factory_impl.dart'; -class DefaultSqliteOpenFactory - extends AbstractDefaultSqliteOpenFactory { - late AbstractDefaultSqliteOpenFactory adapter; +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { + late AbstractDefaultSqliteOpenFactory adapter; DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}) { - adapter = base.DefaultSqliteOpenFactory( - path: path, sqliteOptions: super.sqliteOptions) - as AbstractDefaultSqliteOpenFactory; + adapter = DefaultSqliteOpenFactoryImplementation( + path: path, sqliteOptions: super.sqliteOptions); } @override - FutureOr openDB(SqliteOpenOptions options) { + FutureOr openDB(SqliteOpenOptions options) { return adapter.openDB(options); } @@ -30,7 +27,7 @@ class DefaultSqliteOpenFactory } @override - FutureOr openWeb(SqliteOpenOptions options) { - return adapter.openWeb(options); + FutureOr openExecutor(SqliteOpenOptions options) { + return adapter.openExecutor(options); } } diff --git a/pubspec.yaml b/pubspec.yaml index 7576338..8f19f10 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,6 +12,7 @@ dependencies: async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 + meta: ^1.11.0 dev_dependencies: lints: ^2.0.0 From 14a3fc79180dd20c554e31fa5f7dcf90f906c70b Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 15:01:49 +0200 Subject: [PATCH 16/57] wip isolate connection factory --- lib/src/database/native/connection_pool.dart | 4 +- ...art => native_sqlite_connection_impl.dart} | 0 .../native/native_sqlite_database_impl.dart | 2 +- .../web/executor/drift_sql_executor.dart | 54 ++++++++++++++++ .../web/executor/sqlite_executor.dart | 19 ++++++ lib/src/database/web/web_db_context.dart | 1 + .../web/web_sqlite_connection_impl.dart | 64 +++++++++++++++++++ .../web/web_sqlite_database_impl.dart | 53 +++++++-------- .../native_isolate_connection_factory.dart | 2 +- .../web/web_isolate_connection_factory.dart | 24 +++---- .../open_factory/abstract_open_factory.dart | 13 +--- .../native_sqlite_open_factory_impl.dart | 11 +--- .../stub_sqlite_open_factory.dart | 10 +-- .../web/web_sqlite_open_factory_impl.dart | 63 +++--------------- lib/src/sqlite_connection.dart | 14 ---- 15 files changed, 195 insertions(+), 139 deletions(-) rename lib/src/database/native/{sqlite_connection_impl.dart => native_sqlite_connection_impl.dart} (100%) create mode 100644 lib/src/database/web/executor/drift_sql_executor.dart create mode 100644 lib/src/database/web/executor/sqlite_executor.dart create mode 100644 lib/src/database/web/web_sqlite_connection_impl.dart diff --git a/lib/src/database/native/connection_pool.dart b/lib/src/database/native/connection_pool.dart index 482952b..25c4005 100644 --- a/lib/src/database/native/connection_pool.dart +++ b/lib/src/database/native/connection_pool.dart @@ -8,7 +8,7 @@ import '../../sqlite_open_factory.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; import 'port_channel.dart'; -import 'sqlite_connection_impl.dart'; +import 'native_sqlite_connection_impl.dart'; /// A connection pool with a single write connection and multiple read connections. class SqliteConnectionPool with SqliteQueries implements SqliteConnection { @@ -16,7 +16,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final List _readConnections = []; - final SqliteOpenFactory _factory; + final SqliteOpenFactory _factory; final SerializedPortClient _upstreamPort; @override diff --git a/lib/src/database/native/sqlite_connection_impl.dart b/lib/src/database/native/native_sqlite_connection_impl.dart similarity index 100% rename from lib/src/database/native/sqlite_connection_impl.dart rename to lib/src/database/native/native_sqlite_connection_impl.dart diff --git a/lib/src/database/native/native_sqlite_database_impl.dart b/lib/src/database/native/native_sqlite_database_impl.dart index d8e26f0..f4fb435 100644 --- a/lib/src/database/native/native_sqlite_database_impl.dart +++ b/lib/src/database/native/native_sqlite_database_impl.dart @@ -12,7 +12,7 @@ import '../../update_notification.dart'; import '../abstract_sqlite_database.dart'; import 'port_channel.dart'; import 'connection_pool.dart'; -import 'sqlite_connection_impl.dart'; +import 'native_sqlite_connection_impl.dart'; /// A SQLite database instance. /// diff --git a/lib/src/database/web/executor/drift_sql_executor.dart b/lib/src/database/web/executor/drift_sql_executor.dart new file mode 100644 index 0000000..ae73788 --- /dev/null +++ b/lib/src/database/web/executor/drift_sql_executor.dart @@ -0,0 +1,54 @@ +import 'dart:async'; + +import 'package:drift/drift.dart'; +import 'package:drift/wasm.dart'; +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; + +class DriftWebSQLExecutor extends SQLExecutor { + WasmDatabaseResult db; + + @override + bool closed = false; + + DriftWebSQLExecutor(this.db) { + // Pass on table updates + updateStream = db.resolvedExecutor.streamQueries + .updatesForSync(TableUpdateQuery.any()) + .map((tables) { + return tables.map((e) => e.table).toSet(); + }); + } + + @override + close() { + closed = true; + return db.resolvedExecutor.close(); + } + + @override + Future executeBatch(String sql, List> parameterSets) { + return db.resolvedExecutor.runBatched(BatchedStatements([sql], + parameterSets.map((e) => ArgumentsForBatchedStatement(0, e)).toList())); + } + + @override + FutureOr select(String sql, + [List parameters = const []]) async { + final result = await db.resolvedExecutor.runSelect(sql, parameters); + if (result.isEmpty) { + return ResultSet([], [], []); + } + return ResultSet(result.first.keys.toList(), [], + result.map((e) => e.values.toList()).toList()); + } +} + +class DriftSqliteUser extends QueryExecutorUser { + @override + Future beforeOpen( + QueryExecutor executor, OpeningDetails details) async {} + + @override + int get schemaVersion => 1; +} diff --git a/lib/src/database/web/executor/sqlite_executor.dart b/lib/src/database/web/executor/sqlite_executor.dart new file mode 100644 index 0000000..c6192fa --- /dev/null +++ b/lib/src/database/web/executor/sqlite_executor.dart @@ -0,0 +1,19 @@ +// Abstract class which provides base methods required for Context providers +import 'dart:async'; + +import 'package:sqlite_async/sqlite3_common.dart'; + +/// Abstract class for providing basic SQLite operations +/// Specific DB implementations such as Drift can be adapted to +/// this interface +abstract class SQLExecutor { + bool get closed; + + Stream> updateStream = Stream.empty(); + + Future close(); + + FutureOr select(String sql, [List parameters = const []]); + + FutureOr executeBatch(String sql, List> parameterSets) {} +} diff --git a/lib/src/database/web/web_db_context.dart b/lib/src/database/web/web_db_context.dart index 1067458..24ccb3e 100644 --- a/lib/src/database/web/web_db_context.dart +++ b/lib/src/database/web/web_db_context.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'package:sqlite3/common.dart'; import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; class WebReadContext implements SqliteReadContext { SQLExecutor db; diff --git a/lib/src/database/web/web_sqlite_connection_impl.dart b/lib/src/database/web/web_sqlite_connection_impl.dart new file mode 100644 index 0000000..3163a16 --- /dev/null +++ b/lib/src/database/web/web_sqlite_connection_impl.dart @@ -0,0 +1,64 @@ +import 'dart:async'; +import 'package:meta/meta.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:mutex/mutex.dart'; +import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; +import 'package:sqlite_async/src/database/web/web_db_context.dart'; +import 'package:sqlite_async/src/open_factory/web/web_sqlite_open_factory_impl.dart'; + +class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { + @override + bool get closed { + return executor == null || executor!.closed; + } + + @override + late Stream updates; + + late final Mutex mutex; + DefaultSqliteOpenFactoryImplementation openFactory; + + @protected + final StreamController updatesController = + StreamController.broadcast(); + + @protected + late SQLExecutor? executor; + + @protected + late Future isInitialized; + + WebSqliteConnectionImpl({required this.openFactory, required this.mutex}) { + updates = updatesController.stream; + isInitialized = _init(); + } + + Future _init() async { + executor = await openFactory.openExecutor( + SqliteOpenOptions(primaryConnection: true, readOnly: false)); + + executor!.updateStream.forEach((tables) { + updatesController.add(UpdateNotification(tables)); + }); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) async { + await isInitialized; + return mutex.protect(() => callback(WebReadContext(executor!))); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) async { + await isInitialized; + return mutex.protect(() => callback(WebWriteContext(executor!))); + } + + @override + Future close() async { + await isInitialized; + await executor!.close(); + } +} diff --git a/lib/src/database/web/web_sqlite_database_impl.dart b/lib/src/database/web/web_sqlite_database_impl.dart index 7b1d4f3..dfacc4d 100644 --- a/lib/src/database/web/web_sqlite_database_impl.dart +++ b/lib/src/database/web/web_sqlite_database_impl.dart @@ -1,13 +1,18 @@ import 'dart:async'; -import 'package:meta/meta.dart'; -import 'package:sqlite_async/sqlite_async.dart'; import 'package:mutex/mutex.dart'; -import 'package:sqlite_async/src/database/web/web_db_context.dart'; +import 'package:sqlite_async/src/database/web/web_sqlite_connection_impl.dart'; + +import 'package:sqlite_async/src/isolate_connection_factory/web/web_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_database.dart'; +import 'package:sqlite_async/src/open_factory/web/web_sqlite_open_factory_impl.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/update_notification.dart'; class SqliteDatabaseImplementation extends AbstractSqliteDatabase { @override bool get closed { - return executor == null || executor!.closed; + return _connection.closed; } @override @@ -20,13 +25,11 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { late Future isInitialized; @override - SqliteOpenFactory openFactory; - - late final Future executorFuture; - late Mutex mutex; + DefaultSqliteOpenFactoryImplementation openFactory; - @protected - late SQLExecutor? executor; + late final Mutex mutex; + late final IsolateConnectionFactory _isolateConnectionFactory; + late final WebSqliteConnectionImpl _connection; /// Open a SqliteDatabase. /// @@ -42,8 +45,8 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = - DefaultSqliteOpenFactory(path: path, sqliteOptions: options); + final factory = DefaultSqliteOpenFactoryImplementation( + path: path, sqliteOptions: options); return SqliteDatabaseImplementation.withFactory(factory, maxReaders: maxReaders); } @@ -59,44 +62,42 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { /// 4. Creating temporary views or triggers. SqliteDatabaseImplementation.withFactory(this.openFactory, {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { - executorFuture = openFactory.openExecutor( - SqliteOpenOptions(primaryConnection: true, readOnly: false)) - as Future; updates = updatesController.stream; mutex = Mutex(); + _isolateConnectionFactory = + IsolateConnectionFactory(openFactory: openFactory, mutex: mutex); + _connection = _isolateConnectionFactory.open(); isInitialized = _init(); } Future _init() async { - executor = await executorFuture; - executor!.updateStream.forEach((tables) { - updatesController.add(UpdateNotification(tables)); + await _connection.isInitialized; + _connection.updates.forEach((update) { + updatesController.add(update); }); } @override Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { - await isInitialized; - return mutex.protect(() => callback(WebReadContext(executor!))); + return _connection.readLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { - await isInitialized; - return mutex.protect(() => callback(WebWriteContext(executor!))); + return _connection.writeLock(callback, + lockTimeout: lockTimeout, debugContext: debugContext); } @override Future close() async { - await isInitialized; - await executor!.close(); + return _connection.close(); } @override IsolateConnectionFactory isolateConnectionFactory() { - // TODO: implement isolateConnectionFactory - throw UnimplementedError(); + return _isolateConnectionFactory; } } diff --git a/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart index 67b5aa1..430c83a 100644 --- a/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart @@ -7,7 +7,7 @@ import '../../sqlite_connection.dart'; import '../../update_notification.dart'; import '../../utils/native_database_utils.dart'; import '../../database/native/port_channel.dart'; -import '../../database/native/sqlite_connection_impl.dart'; +import '../../database/native/native_sqlite_connection_impl.dart'; import '../abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. diff --git a/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart b/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart index db47924..3e47d93 100644 --- a/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart @@ -1,26 +1,26 @@ import 'dart:async'; -import 'package:sqlite3/common.dart'; - -import '../../sqlite_connection.dart'; -import '../../sqlite_open_factory.dart'; -import '../abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/src/database/web/web_sqlite_connection_impl.dart'; +import 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/open_factory/web/web_sqlite_open_factory_impl.dart'; +import 'package:sqlite_async/src/sqlite_open_factory.dart'; +import 'package:mutex/mutex.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @override - AbstractDefaultSqliteOpenFactory openFactory; + DefaultSqliteOpenFactoryImplementation openFactory; + + Mutex mutex; - IsolateConnectionFactory({ - required this.openFactory, - }); + IsolateConnectionFactory({required this.openFactory, required this.mutex}); /// Open a new SqliteConnection. /// /// This opens a single connection in a background execution isolate. - SqliteConnection open({String? debugName, bool readOnly = false}) { - // TODO - return {} as SqliteConnection; + WebSqliteConnectionImpl open({String? debugName, bool readOnly = false}) { + return WebSqliteConnectionImpl(mutex: mutex, openFactory: openFactory); } /// Opens a synchronous sqlite.Database directly in the current isolate. diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/open_factory/abstract_open_factory.dart index 5981f4f..fc8c54b 100644 --- a/lib/src/open_factory/abstract_open_factory.dart +++ b/lib/src/open_factory/abstract_open_factory.dart @@ -8,17 +8,10 @@ import '../../definitions.dart'; /// /// Since connections are opened in dedicated background isolates, this class /// must be safe to pass to different isolates. -abstract class SqliteOpenFactory { +abstract class SqliteOpenFactory { String get path; FutureOr open(SqliteOpenOptions options); - - /// This includes a limited set of async only SQL APIs required - /// for SqliteDatabase connections - FutureOr openExecutor(SqliteOpenOptions options) { - throw UnimplementedError(); - } } class SqliteOpenOptions { @@ -49,8 +42,8 @@ class SqliteOpenOptions { /// /// Override the [open] method to customize the process. abstract class AbstractDefaultSqliteOpenFactory< - Database extends sqlite.CommonDatabase, Executor extends SQLExecutor> - implements SqliteOpenFactory { + Database extends sqlite.CommonDatabase> + implements SqliteOpenFactory { final String path; final SqliteOptions sqliteOptions; diff --git a/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart b/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart index 116e006..d15d665 100644 --- a/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart +++ b/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart @@ -1,12 +1,10 @@ -import 'dart:async'; - import 'package:sqlite3/sqlite3.dart'; + import 'package:sqlite_async/src/open_factory/abstract_open_factory.dart'; -import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; class DefaultSqliteOpenFactoryImplementation - extends AbstractDefaultSqliteOpenFactory { + extends AbstractDefaultSqliteOpenFactory { const DefaultSqliteOpenFactoryImplementation( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); @@ -38,9 +36,4 @@ class DefaultSqliteOpenFactoryImplementation } return statements; } - - @override - FutureOr openExecutor(SqliteOpenOptions options) { - throw UnimplementedError(); - } } diff --git a/lib/src/open_factory/stub_sqlite_open_factory.dart b/lib/src/open_factory/stub_sqlite_open_factory.dart index f6047fc..3bdb521 100644 --- a/lib/src/open_factory/stub_sqlite_open_factory.dart +++ b/lib/src/open_factory/stub_sqlite_open_factory.dart @@ -1,7 +1,4 @@ -import 'dart:async'; - -import 'package:sqlite3/common.dart'; -import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; @@ -20,9 +17,4 @@ class DefaultSqliteOpenFactoryImplementation List pragmaStatements(SqliteOpenOptions options) { throw UnimplementedError(); } - - @override - FutureOr openExecutor(SqliteOpenOptions options) { - throw UnimplementedError(); - } } diff --git a/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart b/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart index 0f32739..d44e394 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart +++ b/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart @@ -1,62 +1,14 @@ import 'dart:async'; -import 'package:drift/drift.dart'; import 'package:drift/wasm.dart'; -import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/database/web/executor/drift_sql_executor.dart'; +import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; -import 'package:sqlite3/wasm.dart'; import '../abstract_open_factory.dart'; - -class DriftWebSQLExecutor extends SQLExecutor { - WasmDatabaseResult db; - - @override - bool closed = false; - - DriftWebSQLExecutor(this.db) { - // Pass on table updates - updateStream = db.resolvedExecutor.streamQueries - .updatesForSync(TableUpdateQuery.any()) - .map((tables) { - return tables.map((e) => e.table).toSet(); - }); - } - - @override - close() { - closed = true; - return db.resolvedExecutor.close(); - } - - @override - Future executeBatch(String sql, List> parameterSets) { - return db.resolvedExecutor.runBatched(BatchedStatements([sql], - parameterSets.map((e) => ArgumentsForBatchedStatement(0, e)).toList())); - } - - @override - FutureOr select(String sql, - [List parameters = const []]) async { - final result = await db.resolvedExecutor.runSelect(sql, parameters); - if (result.isEmpty) { - return ResultSet([], [], []); - } - return ResultSet(result.first.keys.toList(), [], - result.map((e) => e.values.toList()).toList()); - } -} - -class SqliteUser extends QueryExecutorUser { - @override - Future beforeOpen( - QueryExecutor executor, OpeningDetails details) async {} - - @override - int get schemaVersion => 1; -} +import 'package:sqlite3/wasm.dart'; class DefaultSqliteOpenFactoryImplementation - extends AbstractDefaultSqliteOpenFactory { + extends AbstractDefaultSqliteOpenFactory { DefaultSqliteOpenFactoryImplementation( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); @@ -77,8 +29,9 @@ class DefaultSqliteOpenFactoryImplementation return wasmSqlite.open(path); } - @override - + /// Returns a simple asynchronous SQLExecutor which can be used to implement + /// higher order functionality. + /// Currently this only uses the Drift WASM implementation. /// The Drift SQLite package provides built in async Web worker functionality /// and automatic persistence storage selection. /// Due to being asynchronous, the under laying CommonDatabase is not accessible @@ -90,7 +43,7 @@ class DefaultSqliteOpenFactoryImplementation ); final executor = DriftWebSQLExecutor(db); - await db.resolvedExecutor.ensureOpen(SqliteUser()); + await db.resolvedExecutor.ensureOpen(DriftSqliteUser()); return executor; } diff --git a/lib/src/sqlite_connection.dart b/lib/src/sqlite_connection.dart index 6a959c7..48f1d39 100644 --- a/lib/src/sqlite_connection.dart +++ b/lib/src/sqlite_connection.dart @@ -2,20 +2,6 @@ import 'dart:async'; import 'package:sqlite3/common.dart' as sqlite; -// Abstract class which provides base methods required for Context providers -abstract class SQLExecutor { - bool get closed; - - Stream> updateStream = Stream.empty(); - - Future close(); - - FutureOr select(String sql, - [List parameters = const []]); - - FutureOr executeBatch(String sql, List> parameterSets) {} -} - /// Abstract class representing calls available in a read-only or read-write context. abstract class SqliteReadContext { /// Execute a read-only (SELECT) query and return the results. From b56e64a6715f906d8565b420907ae0d6311c2fa7 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 16:20:15 +0200 Subject: [PATCH 17/57] wip: organize different implementations --- lib/definitions.dart | 3 + .../abstract_isolate_connection_factory.dart | 4 +- .../abstract_open_factory.dart | 4 +- .../abstract_sqlite_database.dart | 10 +- lib/src/database/sqlite_database_impl.dart | 5 - .../impl/isolate_connection_factory_impl.dart | 5 + lib/src/impl/open_factory_impl.dart | 5 + lib/src/impl/sqlite_database_impl.dart | 5 + .../stub_isolate_connection_factory.dart | 4 +- .../stub_sqlite_database.dart | 18 ++-- .../stub_sqlite_open_factory.dart | 7 +- lib/src/isolate_connection_factory.dart | 27 +----- lib/src/mutex.dart | 2 +- .../database}/connection_pool.dart | 6 +- .../native_sqlite_connection_impl.dart | 8 +- .../database/native_sqlite_database.dart} | 12 +-- .../database}/port_channel.dart | 0 .../native_isolate_connection_factory.dart | 18 ++-- .../native_sqlite_open_factory.dart} | 6 +- lib/src/open_factory/open_factory_impl.dart | 5 - lib/src/sqlite_database.dart | 96 +------------------ lib/src/sqlite_open_factory.dart | 34 +------ .../executor/drift_sql_executor.dart | 2 +- .../database}/executor/sqlite_executor.dart | 0 .../web => web/database}/web_db_context.dart | 6 +- .../web => web/database}/web_locks.dart | 0 .../database}/web_sqlite_connection_impl.dart | 15 ++- .../database/web_sqlite_database.dart} | 24 ++--- .../web/web_isolate_connection_factory.dart | 11 ++- .../web_sqlite_open_factory.dart} | 14 +-- scripts/benchmark.dart | 11 +-- test/close_test.dart | 2 +- test/isolate_test.dart | 3 +- test/json1_test.dart | 2 +- test/util.dart | 2 +- 35 files changed, 120 insertions(+), 256 deletions(-) rename lib/src/{isolate_connection_factory => common}/abstract_isolate_connection_factory.dart (92%) rename lib/src/{open_factory => common}/abstract_open_factory.dart (93%) rename lib/src/{database => common}/abstract_sqlite_database.dart (78%) delete mode 100644 lib/src/database/sqlite_database_impl.dart create mode 100644 lib/src/impl/isolate_connection_factory_impl.dart create mode 100644 lib/src/impl/open_factory_impl.dart create mode 100644 lib/src/impl/sqlite_database_impl.dart rename lib/src/{isolate_connection_factory => impl}/stub_isolate_connection_factory.dart (89%) rename lib/src/{database => impl}/stub_sqlite_database.dart (63%) rename lib/src/{open_factory => impl}/stub_sqlite_open_factory.dart (67%) rename lib/src/{database/native => native/database}/connection_pool.dart (97%) rename lib/src/{database/native => native/database}/native_sqlite_connection_impl.dart (97%) rename lib/src/{database/native/native_sqlite_database_impl.dart => native/database/native_sqlite_database.dart} (94%) rename lib/src/{database/native => native/database}/port_channel.dart (100%) rename lib/src/{isolate_connection_factory => }/native/native_isolate_connection_factory.dart (82%) rename lib/src/{open_factory/native/native_sqlite_open_factory_impl.dart => native/native_sqlite_open_factory.dart} (87%) delete mode 100644 lib/src/open_factory/open_factory_impl.dart rename lib/src/{database/web => web/database}/executor/drift_sql_executor.dart (94%) rename lib/src/{database/web => web/database}/executor/sqlite_executor.dart (100%) rename lib/src/{database/web => web/database}/web_db_context.dart (88%) rename lib/src/{database/web => web/database}/web_locks.dart (100%) rename lib/src/{database/web => web/database}/web_sqlite_connection_impl.dart (78%) rename lib/src/{database/web/web_sqlite_database_impl.dart => web/database/web_sqlite_database.dart} (79%) rename lib/src/{isolate_connection_factory => }/web/web_isolate_connection_factory.dart (76%) rename lib/src/{open_factory/web/web_sqlite_open_factory_impl.dart => web/web_sqlite_open_factory.dart} (86%) diff --git a/lib/definitions.dart b/lib/definitions.dart index 96ecb21..9781377 100644 --- a/lib/definitions.dart +++ b/lib/definitions.dart @@ -3,3 +3,6 @@ export 'package:sqlite_async/src/sqlite_connection.dart'; export 'package:sqlite_async/src/sqlite_queries.dart'; export 'package:sqlite_async/src/sqlite_open_factory.dart'; export 'package:sqlite_async/src/sqlite_options.dart'; +export 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +export 'package:sqlite_async/src/common/abstract_open_factory.dart'; +export 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; diff --git a/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart b/lib/src/common/abstract_isolate_connection_factory.dart similarity index 92% rename from lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart rename to lib/src/common/abstract_isolate_connection_factory.dart index ee40f7f..624a7b0 100644 --- a/lib/src/isolate_connection_factory/abstract_isolate_connection_factory.dart +++ b/lib/src/common/abstract_isolate_connection_factory.dart @@ -1,7 +1,9 @@ import 'dart:async'; -import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/definitions.dart'; +import 'abstract_open_factory.dart'; + /// A connection factory that can be passed to different isolates. abstract class AbstractIsolateConnectionFactory { AbstractDefaultSqliteOpenFactory get openFactory; diff --git a/lib/src/open_factory/abstract_open_factory.dart b/lib/src/common/abstract_open_factory.dart similarity index 93% rename from lib/src/open_factory/abstract_open_factory.dart rename to lib/src/common/abstract_open_factory.dart index fc8c54b..08d79d4 100644 --- a/lib/src/open_factory/abstract_open_factory.dart +++ b/lib/src/common/abstract_open_factory.dart @@ -1,8 +1,8 @@ import 'dart:async'; import 'package:meta/meta.dart'; -import 'package:sqlite3/common.dart' as sqlite; -import '../../definitions.dart'; +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/src/sqlite_options.dart'; /// Factory to create new SQLite database connections. /// diff --git a/lib/src/database/abstract_sqlite_database.dart b/lib/src/common/abstract_sqlite_database.dart similarity index 78% rename from lib/src/database/abstract_sqlite_database.dart rename to lib/src/common/abstract_sqlite_database.dart index 17c84ef..f0d365d 100644 --- a/lib/src/database/abstract_sqlite_database.dart +++ b/lib/src/common/abstract_sqlite_database.dart @@ -1,8 +1,10 @@ import 'dart:async'; -import 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; - -import '../../definitions.dart'; +import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; /// A SQLite database instance. /// @@ -21,7 +23,7 @@ abstract class AbstractSqliteDatabase extends SqliteConnection /// This must be safe to pass to different isolates. /// /// Use a custom class for this to customize the open process. - SqliteOpenFactory get openFactory; + AbstractDefaultSqliteOpenFactory get openFactory; /// Use this stream to subscribe to notifications of updates to tables. Stream get updates; diff --git a/lib/src/database/sqlite_database_impl.dart b/lib/src/database/sqlite_database_impl.dart deleted file mode 100644 index 108f3c2..0000000 --- a/lib/src/database/sqlite_database_impl.dart +++ /dev/null @@ -1,5 +0,0 @@ -export 'stub_sqlite_database.dart' - // ignore: uri_does_not_exist - if (dart.library.io) './native/native_sqlite_database_impl.dart' - // ignore: uri_does_not_exist - if (dart.library.html) './web/web_sqlite_database_impl.dart'; diff --git a/lib/src/impl/isolate_connection_factory_impl.dart b/lib/src/impl/isolate_connection_factory_impl.dart new file mode 100644 index 0000000..4812688 --- /dev/null +++ b/lib/src/impl/isolate_connection_factory_impl.dart @@ -0,0 +1,5 @@ +export 'stub_isolate_connection_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/native_isolate_connection_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.html) '../web/web_isolate_connection_factory.dart'; diff --git a/lib/src/impl/open_factory_impl.dart b/lib/src/impl/open_factory_impl.dart new file mode 100644 index 0000000..e8ce4e1 --- /dev/null +++ b/lib/src/impl/open_factory_impl.dart @@ -0,0 +1,5 @@ +export 'stub_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/native_sqlite_open_factory.dart' + // ignore: uri_does_not_exist + if (dart.library.html) '../web/web_sqlite_open_factory.dart'; diff --git a/lib/src/impl/sqlite_database_impl.dart b/lib/src/impl/sqlite_database_impl.dart new file mode 100644 index 0000000..a2bcd20 --- /dev/null +++ b/lib/src/impl/sqlite_database_impl.dart @@ -0,0 +1,5 @@ +export 'stub_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/database/native_sqlite_database.dart' + // ignore: uri_does_not_exist + if (dart.library.html) '../web/database/web_sqlite_database.dart'; diff --git a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart b/lib/src/impl/stub_isolate_connection_factory.dart similarity index 89% rename from lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart rename to lib/src/impl/stub_isolate_connection_factory.dart index 68d5d88..7182579 100644 --- a/lib/src/isolate_connection_factory/stub_isolate_connection_factory.dart +++ b/lib/src/impl/stub_isolate_connection_factory.dart @@ -2,11 +2,11 @@ import 'dart:async'; import 'package:sqlite3/common.dart'; import 'package:sqlite_async/definitions.dart'; -import 'abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import '../common/abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { - @override AbstractDefaultSqliteOpenFactory openFactory; IsolateConnectionFactory({ diff --git a/lib/src/database/stub_sqlite_database.dart b/lib/src/impl/stub_sqlite_database.dart similarity index 63% rename from lib/src/database/stub_sqlite_database.dart rename to lib/src/impl/stub_sqlite_database.dart index a1f3bf3..ca63288 100644 --- a/lib/src/database/stub_sqlite_database.dart +++ b/lib/src/impl/stub_sqlite_database.dart @@ -1,23 +1,27 @@ -import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/update_notification.dart'; -class SqliteDatabaseImplementation extends AbstractSqliteDatabase { +class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); - @override - SqliteOpenFactory openFactory; + AbstractDefaultSqliteOpenFactory openFactory; @override int maxReaders; - factory SqliteDatabaseImplementation( + factory SqliteDatabase( {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { throw UnimplementedError(); } - SqliteDatabaseImplementation.withFactory(this.openFactory, + SqliteDatabase.withFactory(this.openFactory, {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { throw UnimplementedError(); } @@ -47,7 +51,7 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { } @override - IsolateConnectionFactory isolateConnectionFactory() { + AbstractIsolateConnectionFactory isolateConnectionFactory() { // TODO: implement isolateConnectionFactory throw UnimplementedError(); } diff --git a/lib/src/open_factory/stub_sqlite_open_factory.dart b/lib/src/impl/stub_sqlite_open_factory.dart similarity index 67% rename from lib/src/open_factory/stub_sqlite_open_factory.dart rename to lib/src/impl/stub_sqlite_open_factory.dart index 3bdb521..437ff4b 100644 --- a/lib/src/open_factory/stub_sqlite_open_factory.dart +++ b/lib/src/impl/stub_sqlite_open_factory.dart @@ -1,10 +1,9 @@ import 'package:sqlite_async/sqlite3_common.dart'; -import 'package:sqlite_async/src/sqlite_open_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; -class DefaultSqliteOpenFactoryImplementation - extends AbstractDefaultSqliteOpenFactory { - const DefaultSqliteOpenFactoryImplementation( +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { + const DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); diff --git a/lib/src/isolate_connection_factory.dart b/lib/src/isolate_connection_factory.dart index c108194..a11061d 100644 --- a/lib/src/isolate_connection_factory.dart +++ b/lib/src/isolate_connection_factory.dart @@ -2,29 +2,4 @@ // To conditionally export an implementation for either web or "native" platforms // The sqlite library uses dart:ffi which is not supported on web -import 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; -export 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; - -import '../definitions.dart'; -import './isolate_connection_factory/stub_isolate_connection_factory.dart' as base - if (dart.library.io) './isolate_connection_factory/native/isolate_connection_factory.dart' - if (dart.library.html) './isolate_connection_factory/web/isolate_connection_factory.dart'; - - -class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { - late AbstractIsolateConnectionFactory adapter; - - IsolateConnectionFactory({ - required SqliteOpenFactory openFactory, - }) { - super.openFactory = openFactory; - adapter = base.IsolateConnectionFactory(openFactory: openFactory); - } - - - @override - SqliteConnection open({String? debugName, bool readOnly = false}) { - return adapter.open(debugName: debugName, readOnly: readOnly); - } - -} \ No newline at end of file +export 'impl/isolate_connection_factory_impl.dart'; diff --git a/lib/src/mutex.dart b/lib/src/mutex.dart index 28393a5..4869b03 100644 --- a/lib/src/mutex.dart +++ b/lib/src/mutex.dart @@ -3,7 +3,7 @@ // (MIT) import 'dart:async'; -import 'database/native/port_channel.dart'; +import 'package:sqlite_async/src/native/database/port_channel.dart'; abstract class Mutex { factory Mutex() { diff --git a/lib/src/database/native/connection_pool.dart b/lib/src/native/database/connection_pool.dart similarity index 97% rename from lib/src/database/native/connection_pool.dart rename to lib/src/native/database/connection_pool.dart index 25c4005..0040d9b 100644 --- a/lib/src/database/native/connection_pool.dart +++ b/lib/src/native/database/connection_pool.dart @@ -1,14 +1,12 @@ import 'dart:async'; -import 'package:sqlite3/sqlite3.dart'; - import '../../mutex.dart'; import '../../sqlite_connection.dart'; -import '../../sqlite_open_factory.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; import 'port_channel.dart'; import 'native_sqlite_connection_impl.dart'; +import '../native_sqlite_open_factory.dart'; /// A connection pool with a single write connection and multiple read connections. class SqliteConnectionPool with SqliteQueries implements SqliteConnection { @@ -16,7 +14,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final List _readConnections = []; - final SqliteOpenFactory _factory; + final DefaultSqliteOpenFactory _factory; final SerializedPortClient _upstreamPort; @override diff --git a/lib/src/database/native/native_sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart similarity index 97% rename from lib/src/database/native/native_sqlite_connection_impl.dart rename to lib/src/native/database/native_sqlite_connection_impl.dart index 4ca30e8..a477efa 100644 --- a/lib/src/database/native/native_sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -2,13 +2,13 @@ import 'dart:async'; import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory_impl.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import '../../utils/database_utils.dart'; import '../../mutex.dart'; import 'port_channel.dart'; import '../../sqlite_connection.dart'; -import '../../sqlite_open_factory.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; @@ -50,7 +50,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { return _isolateClient.closed; } - Future _open(DefaultSqliteOpenFactoryImplementation openFactory, + Future _open(DefaultSqliteOpenFactory openFactory, {required bool primary, required SerializedPortClient upstreamPort}) async { await _connectionMutex.lock(() async { @@ -335,7 +335,7 @@ class _SqliteConnectionParams { final SerializedPortClient port; final bool primary; - final DefaultSqliteOpenFactoryImplementation openFactory; + final DefaultSqliteOpenFactory openFactory; _SqliteConnectionParams( {required this.openFactory, diff --git a/lib/src/database/native/native_sqlite_database_impl.dart b/lib/src/native/database/native_sqlite_database.dart similarity index 94% rename from lib/src/database/native/native_sqlite_database_impl.dart rename to lib/src/native/database/native_sqlite_database.dart index f4fb435..cbe9597 100644 --- a/lib/src/database/native/native_sqlite_database_impl.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -1,15 +1,15 @@ import 'dart:async'; import 'dart:isolate'; -import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory_impl.dart'; +import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import '../../../mutex.dart'; import '../../utils/database_utils.dart'; import '../../sqlite_connection.dart'; -import '../../isolate_connection_factory/native/native_isolate_connection_factory.dart'; +import '../native_isolate_connection_factory.dart'; import '../../sqlite_options.dart'; import '../../update_notification.dart'; -import '../abstract_sqlite_database.dart'; +import '../../common/abstract_sqlite_database.dart'; import 'port_channel.dart'; import 'connection_pool.dart'; import 'native_sqlite_connection_impl.dart'; @@ -20,7 +20,7 @@ import 'native_sqlite_connection_impl.dart'; /// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. class SqliteDatabaseImplementation extends AbstractSqliteDatabase { @override - final DefaultSqliteOpenFactoryImplementation openFactory; + final DefaultSqliteOpenFactory openFactory; @override late Stream updates; @@ -53,8 +53,8 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = DefaultSqliteOpenFactoryImplementation( - path: path, sqliteOptions: options); + final factory = + DefaultSqliteOpenFactory(path: path, sqliteOptions: options); return SqliteDatabaseImplementation.withFactory(factory, maxReaders: maxReaders); } diff --git a/lib/src/database/native/port_channel.dart b/lib/src/native/database/port_channel.dart similarity index 100% rename from lib/src/database/native/port_channel.dart rename to lib/src/native/database/port_channel.dart diff --git a/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart similarity index 82% rename from lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart rename to lib/src/native/native_isolate_connection_factory.dart index 430c83a..26fd387 100644 --- a/lib/src/isolate_connection_factory/native/native_isolate_connection_factory.dart +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -1,19 +1,19 @@ import 'dart:async'; import 'dart:isolate'; -import 'package:sqlite_async/src/open_factory/native/native_sqlite_open_factory_impl.dart'; -import '../../mutex.dart'; -import '../../sqlite_connection.dart'; -import '../../update_notification.dart'; -import '../../utils/native_database_utils.dart'; -import '../../database/native/port_channel.dart'; -import '../../database/native/native_sqlite_connection_impl.dart'; -import '../abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; +import '../mutex.dart'; +import '../sqlite_connection.dart'; +import '../update_notification.dart'; +import '../utils/native_database_utils.dart'; +import 'database/port_channel.dart'; +import 'database/native_sqlite_connection_impl.dart'; +import '../common/abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @override - DefaultSqliteOpenFactoryImplementation openFactory; + DefaultSqliteOpenFactory openFactory; SerializedMutex mutex; SerializedPortClient upstreamPort; diff --git a/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart b/lib/src/native/native_sqlite_open_factory.dart similarity index 87% rename from lib/src/open_factory/native/native_sqlite_open_factory_impl.dart rename to lib/src/native/native_sqlite_open_factory.dart index d15d665..83b817c 100644 --- a/lib/src/open_factory/native/native_sqlite_open_factory_impl.dart +++ b/lib/src/native/native_sqlite_open_factory.dart @@ -1,11 +1,11 @@ import 'package:sqlite3/sqlite3.dart'; -import 'package:sqlite_async/src/open_factory/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; -class DefaultSqliteOpenFactoryImplementation +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { - const DefaultSqliteOpenFactoryImplementation( + const DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); diff --git a/lib/src/open_factory/open_factory_impl.dart b/lib/src/open_factory/open_factory_impl.dart deleted file mode 100644 index 9504484..0000000 --- a/lib/src/open_factory/open_factory_impl.dart +++ /dev/null @@ -1,5 +0,0 @@ -export 'stub_sqlite_open_factory.dart' - // ignore: uri_does_not_exist - if (dart.library.io) './native/native_sqlite_open_factory_impl.dart' - // ignore: uri_does_not_exist - if (dart.library.html) './web/web_sqlite_open_factory_impl.dart'; diff --git a/lib/src/sqlite_database.dart b/lib/src/sqlite_database.dart index 40bc032..080cb10 100644 --- a/lib/src/sqlite_database.dart +++ b/lib/src/sqlite_database.dart @@ -1,95 +1 @@ -// This follows the pattern from here: https://stackoverflow.com/questions/58710226/how-to-import-platform-specific-dependency-in-flutter-dart-combine-web-with-an -// To conditionally export an implementation for either web or "native" platforms -// The sqlite library uses dart:ffi which is not supported on web -import 'package:sqlite_async/sqlite_async.dart'; -export 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; -import 'database/sqlite_database_impl.dart'; - -class SqliteDatabase extends AbstractSqliteDatabase { - static const int defaultMaxReaders = AbstractSqliteDatabase.defaultMaxReaders; - - late AbstractSqliteDatabase adapter; - - @override - int get maxReaders { - return adapter.maxReaders; - } - - @override - Future get isInitialized { - return adapter.isInitialized; - } - - @override - SqliteOpenFactory get openFactory { - return adapter.openFactory; - } - - /// Use this stream to subscribe to notifications of updates to tables. - @override - Stream get updates { - return adapter.updates; - } - - /// Open a SqliteDatabase. - /// - /// Only a single SqliteDatabase per [path] should be opened at a time. - /// - /// A connection pool is used by default, allowing multiple concurrent read - /// transactions, and a single concurrent write transaction. Write transactions - /// do not block read transactions, and read transactions will see the state - /// from the last committed write transaction. - /// - /// A maximum of [maxReaders] concurrent read transactions are allowed. - SqliteDatabase( - {required path, - int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, - SqliteOptions options = const SqliteOptions.defaults()}) { - final openFactory = - DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - adapter = SqliteDatabaseImplementation.withFactory(openFactory, - maxReaders: maxReaders); - } - - /// Advanced: Open a database with a specified factory. - /// - /// The factory is used to open each database connection in background isolates. - /// - /// Use when control is required over the opening process. Examples include: - /// 1. Specifying the path to `libsqlite.so` on Linux. - /// 2. Running additional per-connection PRAGMA statements on each connection. - /// 3. Creating custom SQLite functions. - /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(SqliteOpenFactory openFactory, - {int maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { - adapter = SqliteDatabaseImplementation.withFactory(openFactory, - maxReaders: maxReaders); - } - - @override - Future close() { - return adapter.close(); - } - - @override - bool get closed => adapter.closed; - - @override - Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { - return adapter.readLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); - } - - @override - Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) { - return adapter.writeLock(callback, - lockTimeout: lockTimeout, debugContext: debugContext); - } - - @override - AbstractIsolateConnectionFactory isolateConnectionFactory() { - return adapter.isolateConnectionFactory(); - } -} +export 'package:sqlite_async/src/impl/sqlite_database_impl.dart'; diff --git a/lib/src/sqlite_open_factory.dart b/lib/src/sqlite_open_factory.dart index 73167b9..0fd42cb 100644 --- a/lib/src/sqlite_open_factory.dart +++ b/lib/src/sqlite_open_factory.dart @@ -1,33 +1 @@ -export './open_factory/abstract_open_factory.dart'; - -import 'dart:async'; -import 'package:sqlite3/common.dart'; -import 'package:sqlite_async/definitions.dart'; - -import 'open_factory/open_factory_impl.dart'; - -class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { - late AbstractDefaultSqliteOpenFactory adapter; - - DefaultSqliteOpenFactory( - {required super.path, - super.sqliteOptions = const SqliteOptions.defaults()}) { - adapter = DefaultSqliteOpenFactoryImplementation( - path: path, sqliteOptions: super.sqliteOptions); - } - - @override - FutureOr openDB(SqliteOpenOptions options) { - return adapter.openDB(options); - } - - @override - List pragmaStatements(SqliteOpenOptions options) { - return adapter.pragmaStatements(options); - } - - @override - FutureOr openExecutor(SqliteOpenOptions options) { - return adapter.openExecutor(options); - } -} +export 'impl/open_factory_impl.dart'; diff --git a/lib/src/database/web/executor/drift_sql_executor.dart b/lib/src/web/database/executor/drift_sql_executor.dart similarity index 94% rename from lib/src/database/web/executor/drift_sql_executor.dart rename to lib/src/web/database/executor/drift_sql_executor.dart index ae73788..9ff5f4c 100644 --- a/lib/src/database/web/executor/drift_sql_executor.dart +++ b/lib/src/web/database/executor/drift_sql_executor.dart @@ -3,7 +3,7 @@ import 'dart:async'; import 'package:drift/drift.dart'; import 'package:drift/wasm.dart'; import 'package:sqlite3/common.dart'; -import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; +import 'sqlite_executor.dart'; class DriftWebSQLExecutor extends SQLExecutor { WasmDatabaseResult db; diff --git a/lib/src/database/web/executor/sqlite_executor.dart b/lib/src/web/database/executor/sqlite_executor.dart similarity index 100% rename from lib/src/database/web/executor/sqlite_executor.dart rename to lib/src/web/database/executor/sqlite_executor.dart diff --git a/lib/src/database/web/web_db_context.dart b/lib/src/web/database/web_db_context.dart similarity index 88% rename from lib/src/database/web/web_db_context.dart rename to lib/src/web/database/web_db_context.dart index 24ccb3e..74160e3 100644 --- a/lib/src/database/web/web_db_context.dart +++ b/lib/src/web/database/web_db_context.dart @@ -1,8 +1,8 @@ import 'dart:async'; -import 'package:sqlite3/common.dart'; -import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'executor/sqlite_executor.dart'; class WebReadContext implements SqliteReadContext { SQLExecutor db; diff --git a/lib/src/database/web/web_locks.dart b/lib/src/web/database/web_locks.dart similarity index 100% rename from lib/src/database/web/web_locks.dart rename to lib/src/web/database/web_locks.dart diff --git a/lib/src/database/web/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart similarity index 78% rename from lib/src/database/web/web_sqlite_connection_impl.dart rename to lib/src/web/database/web_sqlite_connection_impl.dart index 3163a16..99add43 100644 --- a/lib/src/database/web/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -1,10 +1,15 @@ import 'dart:async'; import 'package:meta/meta.dart'; -import 'package:sqlite_async/sqlite_async.dart'; import 'package:mutex/mutex.dart'; -import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; -import 'package:sqlite_async/src/database/web/web_db_context.dart'; -import 'package:sqlite_async/src/open_factory/web/web_sqlite_open_factory_impl.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; + +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; + +import 'executor/sqlite_executor.dart'; +import 'web_db_context.dart'; class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { @override @@ -16,7 +21,7 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { late Stream updates; late final Mutex mutex; - DefaultSqliteOpenFactoryImplementation openFactory; + DefaultSqliteOpenFactory openFactory; @protected final StreamController updatesController = diff --git a/lib/src/database/web/web_sqlite_database_impl.dart b/lib/src/web/database/web_sqlite_database.dart similarity index 79% rename from lib/src/database/web/web_sqlite_database_impl.dart rename to lib/src/web/database/web_sqlite_database.dart index dfacc4d..2b39ea8 100644 --- a/lib/src/database/web/web_sqlite_database_impl.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -1,15 +1,16 @@ import 'dart:async'; import 'package:mutex/mutex.dart'; -import 'package:sqlite_async/src/database/web/web_sqlite_connection_impl.dart'; -import 'package:sqlite_async/src/isolate_connection_factory/web/web_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; -import 'package:sqlite_async/src/sqlite_database.dart'; -import 'package:sqlite_async/src/open_factory/web/web_sqlite_open_factory_impl.dart'; +import 'package:sqlite_async/src/web/web_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/update_notification.dart'; -class SqliteDatabaseImplementation extends AbstractSqliteDatabase { +import 'web_sqlite_connection_impl.dart'; + +class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed { return _connection.closed; @@ -25,7 +26,7 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { late Future isInitialized; @override - DefaultSqliteOpenFactoryImplementation openFactory; + DefaultSqliteOpenFactory openFactory; late final Mutex mutex; late final IsolateConnectionFactory _isolateConnectionFactory; @@ -41,14 +42,13 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { /// from the last committed write transaction. /// /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabaseImplementation( + factory SqliteDatabase( {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { - final factory = DefaultSqliteOpenFactoryImplementation( - path: path, sqliteOptions: options); - return SqliteDatabaseImplementation.withFactory(factory, - maxReaders: maxReaders); + final factory = + DefaultSqliteOpenFactory(path: path, sqliteOptions: options); + return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -60,7 +60,7 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabaseImplementation.withFactory(this.openFactory, + SqliteDatabase.withFactory(this.openFactory, {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { updates = updatesController.stream; mutex = Mutex(); diff --git a/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart similarity index 76% rename from lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart rename to lib/src/web/web_isolate_connection_factory.dart index 3e47d93..b7d8014 100644 --- a/lib/src/isolate_connection_factory/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -1,16 +1,17 @@ import 'dart:async'; import 'package:sqlite_async/sqlite3_common.dart'; -import 'package:sqlite_async/src/database/web/web_sqlite_connection_impl.dart'; -import 'package:sqlite_async/src/isolate_connection_factory/abstract_isolate_connection_factory.dart'; -import 'package:sqlite_async/src/open_factory/web/web_sqlite_open_factory_impl.dart'; -import 'package:sqlite_async/src/sqlite_open_factory.dart'; +import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'package:mutex/mutex.dart'; +import 'database/web_sqlite_connection_impl.dart'; + /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @override - DefaultSqliteOpenFactoryImplementation openFactory; + DefaultSqliteOpenFactory openFactory; Mutex mutex; diff --git a/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart b/lib/src/web/web_sqlite_open_factory.dart similarity index 86% rename from lib/src/open_factory/web/web_sqlite_open_factory_impl.dart rename to lib/src/web/web_sqlite_open_factory.dart index d44e394..50f9df9 100644 --- a/lib/src/open_factory/web/web_sqlite_open_factory_impl.dart +++ b/lib/src/web/web_sqlite_open_factory.dart @@ -1,15 +1,17 @@ import 'dart:async'; import 'package:drift/wasm.dart'; -import 'package:sqlite_async/src/database/web/executor/drift_sql_executor.dart'; -import 'package:sqlite_async/src/database/web/executor/sqlite_executor.dart'; -import 'package:sqlite_async/src/sqlite_options.dart'; -import '../abstract_open_factory.dart'; import 'package:sqlite3/wasm.dart'; -class DefaultSqliteOpenFactoryImplementation +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; + +import 'database/executor/drift_sql_executor.dart'; +import 'database/executor/sqlite_executor.dart'; + +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { - DefaultSqliteOpenFactoryImplementation( + DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); diff --git a/scripts/benchmark.dart b/scripts/benchmark.dart index 6e8b3d8..a93fea1 100644 --- a/scripts/benchmark.dart +++ b/scripts/benchmark.dart @@ -5,8 +5,6 @@ import 'dart:math'; import 'package:benchmarking/benchmarking.dart'; import 'package:collection/collection.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database/native/native_sqlite_database.dart' - as native_sqlite_database; import '../test/util.dart'; @@ -49,8 +47,7 @@ List benchmarks = [ }, maxBatchSize: 200000, enabled: false), SqliteBenchmark('writeLock in isolate', (SqliteDatabase db, List> parameters) async { - var factory = (db as native_sqlite_database.SqliteDatabase) - .isolateConnectionFactory(); + var factory = db.isolateConnectionFactory(); var len = parameters.length; await Isolate.run(() async { final db = factory.open(); @@ -90,8 +87,7 @@ List benchmarks = [ }, maxBatchSize: 1000), SqliteBenchmark('Insert: executeBatch in isolate', (SqliteDatabase db, List> parameters) async { - var factory = (db as native_sqlite_database.SqliteDatabase) - .isolateConnectionFactory(); + var factory = db.isolateConnectionFactory(); await Isolate.run(() async { final db = factory.open(); await db.executeBatch( @@ -101,8 +97,7 @@ List benchmarks = [ }, maxBatchSize: 20000, enabled: true), SqliteBenchmark('Insert: direct write in isolate', (SqliteDatabase db, List> parameters) async { - var factory = (db as native_sqlite_database.SqliteDatabase) - .isolateConnectionFactory(); + var factory = db.isolateConnectionFactory(); await Isolate.run(() async { final db = factory.open(); for (var params in parameters) { diff --git a/test/close_test.dart b/test/close_test.dart index d29519f..e6a4d74 100644 --- a/test/close_test.dart +++ b/test/close_test.dart @@ -1,7 +1,7 @@ import 'dart:io'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; diff --git a/test/isolate_test.dart b/test/isolate_test.dart index ca71e7a..69b5035 100644 --- a/test/isolate_test.dart +++ b/test/isolate_test.dart @@ -1,7 +1,6 @@ import 'dart:isolate'; import 'package:test/test.dart'; -import 'package:sqlite_async/src/database/native/native_sqlite_database.dart'; import 'util.dart'; @@ -19,7 +18,7 @@ void main() { }); test('Basic Isolate usage', () async { - final db = await setupDatabase(path: path) as SqliteDatabase; + final db = await setupDatabase(path: path); final factory = db.isolateConnectionFactory(); final result = await Isolate.run(() async { diff --git a/test/json1_test.dart b/test/json1_test.dart index 5aa56a1..8443f5b 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -1,5 +1,5 @@ import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/database/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; diff --git a/test/util.dart b/test/util.dart index 79bd81b..bdedb35 100644 --- a/test/util.dart +++ b/test/util.dart @@ -51,7 +51,7 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { } } -SqliteOpenFactory testFactory({String? path}) { +AbstractDefaultSqliteOpenFactory testFactory({String? path}) { return TestSqliteOpenFactory(path: path ?? dbPath()); } From 4327244f089d6f0cd51f631a2cd02a1de281a031 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 16:45:09 +0200 Subject: [PATCH 18/57] cleanup tests --- .../native/database/native_sqlite_connection_impl.dart | 7 ++++--- lib/src/native/database/native_sqlite_database.dart | 9 ++++----- lib/src/native/native_sqlite_open_factory.dart | 6 +++--- test/util.dart | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/src/native/database/native_sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart index a477efa..eb9c446 100644 --- a/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; @@ -12,7 +13,7 @@ import '../../sqlite_connection.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; -typedef TxCallback = Future Function(sqlite.Database db); +typedef TxCallback = Future Function(CommonDatabase db); /// Implements a SqliteConnection using a separate isolate for the database /// operations. @@ -179,7 +180,7 @@ class _TransactionContext implements SqliteWriteContext { @override Future computeWithDatabase( - Future Function(sqlite.Database db) compute) async { + Future Function(CommonDatabase db) compute) async { return _sendPort.post(_SqliteIsolateClosure(compute)); } @@ -239,7 +240,7 @@ void _sqliteConnectionIsolate(_SqliteConnectionParams params) async { } Future _sqliteConnectionIsolateInner(_SqliteConnectionParams params, - ChildPortClient client, sqlite.Database db) async { + ChildPortClient client, CommonDatabase db) async { final server = params.portServer; final commandPort = ReceivePort(); diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index cbe9597..147a9d8 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -18,7 +18,7 @@ import 'native_sqlite_connection_impl.dart'; /// /// Use one instance per database file. If multiple instances are used, update /// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. -class SqliteDatabaseImplementation extends AbstractSqliteDatabase { +class SqliteDatabase extends AbstractSqliteDatabase { @override final DefaultSqliteOpenFactory openFactory; @@ -49,14 +49,13 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { /// from the last committed write transaction. /// /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabaseImplementation( + factory SqliteDatabase( {required path, int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { final factory = DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabaseImplementation.withFactory(factory, - maxReaders: maxReaders); + return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -68,7 +67,7 @@ class SqliteDatabaseImplementation extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabaseImplementation.withFactory(this.openFactory, + SqliteDatabase.withFactory(this.openFactory, {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { updates = updatesController.stream; diff --git a/lib/src/native/native_sqlite_open_factory.dart b/lib/src/native/native_sqlite_open_factory.dart index 83b817c..19fc700 100644 --- a/lib/src/native/native_sqlite_open_factory.dart +++ b/lib/src/native/native_sqlite_open_factory.dart @@ -1,16 +1,16 @@ import 'package:sqlite3/sqlite3.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; -class DefaultSqliteOpenFactory - extends AbstractDefaultSqliteOpenFactory { +class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { const DefaultSqliteOpenFactory( {required super.path, super.sqliteOptions = const SqliteOptions.defaults()}); @override - Database openDB(SqliteOpenOptions options) { + CommonDatabase openDB(SqliteOpenOptions options) { final mode = options.openMode; var db = sqlite3.open(path, mode: mode, mutex: false); return db; diff --git a/test/util.dart b/test/util.dart index bdedb35..f14ae9e 100644 --- a/test/util.dart +++ b/test/util.dart @@ -51,7 +51,7 @@ class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { } } -AbstractDefaultSqliteOpenFactory testFactory({String? path}) { +DefaultSqliteOpenFactory testFactory({String? path}) { return TestSqliteOpenFactory(path: path ?? dbPath()); } From 8e7b1ff0702a95a241213cf9c1d1a7454b0cca8c Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 17:00:41 +0200 Subject: [PATCH 19/57] ignore vscode --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 295944b..0a5c131 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ pubspec.lock .idea +.vscode *.db *.db-* test-db From b0337940ce7cb091ebefd0acd658a6b8db988ea4 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 25 Jan 2024 17:26:02 +0200 Subject: [PATCH 20/57] cleanup mutexes --- lib/src/common/abstract_mutex.dart | 20 ++ lib/src/impl/mutex_impl.dart | 5 + lib/src/impl/stub_mutex.dart | 13 + lib/src/mutex.dart | 270 +----------------- lib/src/native/database/connection_pool.dart | 4 +- .../native_sqlite_connection_impl.dart | 2 +- .../database/native_sqlite_database.dart | 2 +- .../native_isolate_connection_factory.dart | 2 +- lib/src/native/native_isolate_mutex.dart | 250 ++++++++++++++++ .../database/web_sqlite_connection_impl.dart | 6 +- lib/src/web/database/web_sqlite_database.dart | 2 +- .../web/web_isolate_connection_factory.dart | 3 +- lib/src/web/web_mutex.dart | 21 ++ test/mutex_test.dart | 2 +- 14 files changed, 323 insertions(+), 279 deletions(-) create mode 100644 lib/src/common/abstract_mutex.dart create mode 100644 lib/src/impl/mutex_impl.dart create mode 100644 lib/src/impl/stub_mutex.dart create mode 100644 lib/src/native/native_isolate_mutex.dart create mode 100644 lib/src/web/web_mutex.dart diff --git a/lib/src/common/abstract_mutex.dart b/lib/src/common/abstract_mutex.dart new file mode 100644 index 0000000..becd581 --- /dev/null +++ b/lib/src/common/abstract_mutex.dart @@ -0,0 +1,20 @@ +abstract class AbstractMutex { + /// timeout is a timeout for acquiring the lock, not for the callback + Future lock(Future Function() callback, {Duration? timeout}); + + /// Release resources used by the Mutex. + /// + /// Subsequent calls to [lock] may fail, or may never call the callback. + Future close(); +} + +class LockError extends Error { + final String message; + + LockError(this.message); + + @override + String toString() { + return 'LockError: $message'; + } +} diff --git a/lib/src/impl/mutex_impl.dart b/lib/src/impl/mutex_impl.dart new file mode 100644 index 0000000..6b77de7 --- /dev/null +++ b/lib/src/impl/mutex_impl.dart @@ -0,0 +1,5 @@ +export 'stub_mutex.dart' + // ignore: uri_does_not_exist + if (dart.library.io) '../native/native_isolate_mutex.dart' + // ignore: uri_does_not_exist + if (dart.library.html) '../web/web_mutex.dart'; diff --git a/lib/src/impl/stub_mutex.dart b/lib/src/impl/stub_mutex.dart new file mode 100644 index 0000000..96ae62f --- /dev/null +++ b/lib/src/impl/stub_mutex.dart @@ -0,0 +1,13 @@ +import 'package:sqlite_async/src/common/abstract_mutex.dart'; + +class Mutex extends AbstractMutex { + @override + Future close() { + throw UnimplementedError(); + } + + @override + Future lock(Future Function() callback, {Duration? timeout}) { + throw UnimplementedError(); + } +} diff --git a/lib/src/mutex.dart b/lib/src/mutex.dart index 4869b03..8baa5ea 100644 --- a/lib/src/mutex.dart +++ b/lib/src/mutex.dart @@ -1,268 +1,2 @@ -// Adapted from: -// https://github.com/tekartik/synchronized.dart -// (MIT) -import 'dart:async'; - -import 'package:sqlite_async/src/native/database/port_channel.dart'; - -abstract class Mutex { - factory Mutex() { - return SimpleMutex(); - } - - /// timeout is a timeout for acquiring the lock, not for the callback - Future lock(Future Function() callback, {Duration? timeout}); - - /// Release resources used by the Mutex. - /// - /// Subsequent calls to [lock] may fail, or may never call the callback. - Future close(); -} - -/// Mutex maintains a queue of Future-returning functions that -/// are executed sequentially. -/// The internal lock is not shared across Isolates by default. -class SimpleMutex implements Mutex { - // Adapted from https://github.com/tekartik/synchronized.dart/blob/master/synchronized/lib/src/basic_lock.dart - - Future? last; - - // Hack to make sure the Mutex is not copied to another isolate. - // ignore: unused_field - final Finalizer _f = Finalizer((_) {}); - - SimpleMutex(); - - bool get locked => last != null; - - _SharedMutexServer? _shared; - - @override - Future lock(Future Function() callback, {Duration? timeout}) async { - if (Zone.current[this] != null) { - throw LockError('Recursive lock is not allowed'); - } - var zone = Zone.current.fork(zoneValues: {this: true}); - - return zone.run(() async { - final prev = last; - final completer = Completer.sync(); - last = completer.future; - try { - // If there is a previous running block, wait for it - if (prev != null) { - if (timeout != null) { - // This could throw a timeout error - try { - await prev.timeout(timeout); - } catch (error) { - if (error is TimeoutException) { - throw TimeoutException('Failed to acquire lock', timeout); - } else { - rethrow; - } - } - } else { - await prev; - } - } - - // Run the function and return the result - return await callback(); - } finally { - // Cleanup - // waiting for the previous task to be done in case of timeout - void complete() { - // Only mark it unlocked when the last one complete - if (identical(last, completer.future)) { - last = null; - } - completer.complete(); - } - - // In case of timeout, wait for the previous one to complete too - // before marking this task as complete - if (prev != null && timeout != null) { - // But we still returns immediately - prev.then((_) { - complete(); - }).ignore(); - } else { - complete(); - } - } - }); - } - - @override - Future close() async { - _shared?.close(); - await lock(() async {}); - } - - /// Get a serialized instance that can be passed over to a different isolate. - SerializedMutex get shared { - _shared ??= _SharedMutexServer._withMutex(this); - return _shared!.serialized; - } -} - -/// Serialized version of a Mutex, can be passed over to different isolates. -/// Use [open] to get a [SharedMutex] instance. -/// -/// Uses a [SendPort] to communicate with the source mutex. -class SerializedMutex { - final SerializedPortClient client; - - const SerializedMutex(this.client); - - SharedMutex open() { - return SharedMutex._(client.open()); - } -} - -/// Mutex instantiated from a source mutex, potentially in a different isolate. -/// -/// Uses a [SendPort] to communicate with the source mutex. -class SharedMutex implements Mutex { - final ChildPortClient client; - bool closed = false; - - SharedMutex._(this.client); - - @override - Future lock(Future Function() callback, {Duration? timeout}) async { - if (Zone.current[this] != null) { - throw LockError('Recursive lock is not allowed'); - } - return runZoned(() async { - if (closed) { - throw const ClosedException(); - } - await _acquire(timeout: timeout); - try { - final T result = await callback(); - return result; - } finally { - _unlock(); - } - }, zoneValues: {this: true}); - } - - _unlock() { - client.fire(const _UnlockMessage()); - } - - Future _acquire({Duration? timeout}) async { - final lockFuture = client.post(const _AcquireMessage()); - bool timedout = false; - - var handledLockFuture = lockFuture.then((_) { - if (timedout) { - _unlock(); - throw TimeoutException('Failed to acquire lock', timeout); - } - }); - - if (timeout != null) { - handledLockFuture = - handledLockFuture.timeout(timeout).catchError((error, stacktrace) { - timedout = true; - if (error is TimeoutException) { - throw TimeoutException('Failed to acquire SharedMutex lock', timeout); - } - throw error; - }); - } - return await handledLockFuture; - } - - @override - - /// Wait for existing locks to be released, then close this SharedMutex - /// and prevent further locks from being taken out. - Future close() async { - if (closed) { - return; - } - closed = true; - // Wait for any existing locks to complete, then prevent any further locks from being taken out. - await _acquire(); - client.fire(const _CloseMessage()); - // Close client immediately after _unlock(), - // so that we're sure no further locks are acquired. - // This also cancels any lock request in process. - client.close(); - } -} - -/// Manages a [SerializedMutex], allowing a [Mutex] to be shared across isolates. -class _SharedMutexServer { - Completer? unlock; - late final SerializedMutex serialized; - final Mutex mutex; - bool closed = false; - - late final PortServer server; - - _SharedMutexServer._withMutex(this.mutex) { - server = PortServer((Object? arg) async { - return await _handle(arg); - }); - serialized = SerializedMutex(server.client()); - } - - Future _handle(Object? arg) async { - if (arg is _AcquireMessage) { - var lock = Completer.sync(); - mutex.lock(() async { - if (closed) { - // The client will error already - we just need to ensure - // we don't take out another lock. - return; - } - assert(unlock == null); - unlock = Completer.sync(); - lock.complete(); - await unlock!.future; - unlock = null; - }); - await lock.future; - } else if (arg is _UnlockMessage) { - assert(unlock != null); - unlock!.complete(); - } else if (arg is _CloseMessage) { - // Unlock and close (from client side) - closed = true; - unlock?.complete(); - } - } - - void close() async { - server.close(); - } -} - -class _AcquireMessage { - const _AcquireMessage(); -} - -class _UnlockMessage { - const _UnlockMessage(); -} - -/// Unlock and close -class _CloseMessage { - const _CloseMessage(); -} - -class LockError extends Error { - final String message; - - LockError(this.message); - - @override - String toString() { - return 'LockError: $message'; - } -} +export 'impl/mutex_impl.dart'; +export 'common/abstract_mutex.dart'; diff --git a/lib/src/native/database/connection_pool.dart b/lib/src/native/database/connection_pool.dart index 0040d9b..b93a30f 100644 --- a/lib/src/native/database/connection_pool.dart +++ b/lib/src/native/database/connection_pool.dart @@ -1,6 +1,8 @@ import 'dart:async'; -import '../../mutex.dart'; +import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; + import '../../sqlite_connection.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; diff --git a/lib/src/native/database/native_sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart index eb9c446..041cdae 100644 --- a/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -4,10 +4,10 @@ import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import '../../utils/database_utils.dart'; -import '../../mutex.dart'; import 'port_channel.dart'; import '../../sqlite_connection.dart'; import '../../sqlite_queries.dart'; diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index 147a9d8..ba5573e 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -1,9 +1,9 @@ import 'dart:async'; import 'dart:isolate'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; -import '../../../mutex.dart'; import '../../utils/database_utils.dart'; import '../../sqlite_connection.dart'; import '../native_isolate_connection_factory.dart'; diff --git a/lib/src/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart index 26fd387..2e9a01e 100644 --- a/lib/src/native/native_isolate_connection_factory.dart +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -1,8 +1,8 @@ import 'dart:async'; import 'dart:isolate'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; -import '../mutex.dart'; import '../sqlite_connection.dart'; import '../update_notification.dart'; import '../utils/native_database_utils.dart'; diff --git a/lib/src/native/native_isolate_mutex.dart b/lib/src/native/native_isolate_mutex.dart new file mode 100644 index 0000000..a252a67 --- /dev/null +++ b/lib/src/native/native_isolate_mutex.dart @@ -0,0 +1,250 @@ +// Adapted from: +// https://github.com/tekartik/synchronized.dart +// (MIT) +import 'dart:async'; + +import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/native/database/port_channel.dart'; + +abstract class Mutex extends AbstractMutex { + factory Mutex() { + return SimpleMutex(); + } +} + +/// Mutex maintains a queue of Future-returning functions that +/// are executed sequentially. +/// The internal lock is not shared across Isolates by default. +class SimpleMutex implements Mutex { + // Adapted from https://github.com/tekartik/synchronized.dart/blob/master/synchronized/lib/src/basic_lock.dart + + Future? last; + + // Hack to make sure the Mutex is not copied to another isolate. + // ignore: unused_field + final Finalizer _f = Finalizer((_) {}); + + SimpleMutex(); + + bool get locked => last != null; + + _SharedMutexServer? _shared; + + @override + Future lock(Future Function() callback, {Duration? timeout}) async { + if (Zone.current[this] != null) { + throw LockError('Recursive lock is not allowed'); + } + var zone = Zone.current.fork(zoneValues: {this: true}); + + return zone.run(() async { + final prev = last; + final completer = Completer.sync(); + last = completer.future; + try { + // If there is a previous running block, wait for it + if (prev != null) { + if (timeout != null) { + // This could throw a timeout error + try { + await prev.timeout(timeout); + } catch (error) { + if (error is TimeoutException) { + throw TimeoutException('Failed to acquire lock', timeout); + } else { + rethrow; + } + } + } else { + await prev; + } + } + + // Run the function and return the result + return await callback(); + } finally { + // Cleanup + // waiting for the previous task to be done in case of timeout + void complete() { + // Only mark it unlocked when the last one complete + if (identical(last, completer.future)) { + last = null; + } + completer.complete(); + } + + // In case of timeout, wait for the previous one to complete too + // before marking this task as complete + if (prev != null && timeout != null) { + // But we still returns immediately + prev.then((_) { + complete(); + }).ignore(); + } else { + complete(); + } + } + }); + } + + @override + Future close() async { + _shared?.close(); + await lock(() async {}); + } + + /// Get a serialized instance that can be passed over to a different isolate. + SerializedMutex get shared { + _shared ??= _SharedMutexServer._withMutex(this); + return _shared!.serialized; + } +} + +/// Serialized version of a Mutex, can be passed over to different isolates. +/// Use [open] to get a [SharedMutex] instance. +/// +/// Uses a [SendPort] to communicate with the source mutex. +class SerializedMutex { + final SerializedPortClient client; + + const SerializedMutex(this.client); + + SharedMutex open() { + return SharedMutex._(client.open()); + } +} + +/// Mutex instantiated from a source mutex, potentially in a different isolate. +/// +/// Uses a [SendPort] to communicate with the source mutex. +class SharedMutex implements Mutex { + final ChildPortClient client; + bool closed = false; + + SharedMutex._(this.client); + + @override + Future lock(Future Function() callback, {Duration? timeout}) async { + if (Zone.current[this] != null) { + throw LockError('Recursive lock is not allowed'); + } + return runZoned(() async { + if (closed) { + throw const ClosedException(); + } + await _acquire(timeout: timeout); + try { + final T result = await callback(); + return result; + } finally { + _unlock(); + } + }, zoneValues: {this: true}); + } + + _unlock() { + client.fire(const _UnlockMessage()); + } + + Future _acquire({Duration? timeout}) async { + final lockFuture = client.post(const _AcquireMessage()); + bool timedout = false; + + var handledLockFuture = lockFuture.then((_) { + if (timedout) { + _unlock(); + throw TimeoutException('Failed to acquire lock', timeout); + } + }); + + if (timeout != null) { + handledLockFuture = + handledLockFuture.timeout(timeout).catchError((error, stacktrace) { + timedout = true; + if (error is TimeoutException) { + throw TimeoutException('Failed to acquire SharedMutex lock', timeout); + } + throw error; + }); + } + return await handledLockFuture; + } + + @override + + /// Wait for existing locks to be released, then close this SharedMutex + /// and prevent further locks from being taken out. + Future close() async { + if (closed) { + return; + } + closed = true; + // Wait for any existing locks to complete, then prevent any further locks from being taken out. + await _acquire(); + client.fire(const _CloseMessage()); + // Close client immediately after _unlock(), + // so that we're sure no further locks are acquired. + // This also cancels any lock request in process. + client.close(); + } +} + +/// Manages a [SerializedMutex], allowing a [Mutex] to be shared across isolates. +class _SharedMutexServer { + Completer? unlock; + late final SerializedMutex serialized; + final Mutex mutex; + bool closed = false; + + late final PortServer server; + + _SharedMutexServer._withMutex(this.mutex) { + server = PortServer((Object? arg) async { + return await _handle(arg); + }); + serialized = SerializedMutex(server.client()); + } + + Future _handle(Object? arg) async { + if (arg is _AcquireMessage) { + var lock = Completer.sync(); + mutex.lock(() async { + if (closed) { + // The client will error already - we just need to ensure + // we don't take out another lock. + return; + } + assert(unlock == null); + unlock = Completer.sync(); + lock.complete(); + await unlock!.future; + unlock = null; + }); + await lock.future; + } else if (arg is _UnlockMessage) { + assert(unlock != null); + unlock!.complete(); + } else if (arg is _CloseMessage) { + // Unlock and close (from client side) + closed = true; + unlock?.complete(); + } + } + + void close() async { + server.close(); + } +} + +class _AcquireMessage { + const _AcquireMessage(); +} + +class _UnlockMessage { + const _UnlockMessage(); +} + +/// Unlock and close +class _CloseMessage { + const _CloseMessage(); +} diff --git a/lib/src/web/database/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart index 99add43..6d93f1f 100644 --- a/lib/src/web/database/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -1,11 +1,11 @@ import 'dart:async'; import 'package:meta/meta.dart'; -import 'package:mutex/mutex.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/web/web_mutex.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'executor/sqlite_executor.dart'; @@ -51,14 +51,14 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.protect(() => callback(WebReadContext(executor!))); + return mutex.lock(() => callback(WebReadContext(executor!))); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.protect(() => callback(WebWriteContext(executor!))); + return mutex.lock(() => callback(WebWriteContext(executor!))); } @override diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index 2b39ea8..1eec9c3 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -1,9 +1,9 @@ import 'dart:async'; -import 'package:mutex/mutex.dart'; import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/web/web_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/web/web_mutex.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/update_notification.dart'; diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index b7d8014..502b7ee 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -4,8 +4,7 @@ import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; -import 'package:mutex/mutex.dart'; - +import 'web_mutex.dart'; import 'database/web_sqlite_connection_impl.dart'; /// A connection factory that can be passed to different isolates. diff --git a/lib/src/web/web_mutex.dart b/lib/src/web/web_mutex.dart new file mode 100644 index 0000000..7c95762 --- /dev/null +++ b/lib/src/web/web_mutex.dart @@ -0,0 +1,21 @@ +import 'package:mutex/mutex.dart' as mutex; +import 'package:sqlite_async/src/mutex.dart'; + +class Mutex extends AbstractMutex { + late final mutex.Mutex m; + + Mutex() { + m = mutex.Mutex(); + } + + @override + Future close() async { + // TODO + } + + @override + Future lock(Future Function() callback, {Duration? timeout}) { + // TODO: use web navigator locks here + return m.protect(callback); + } +} diff --git a/test/mutex_test.dart b/test/mutex_test.dart index 0eb90e1..77561e5 100644 --- a/test/mutex_test.dart +++ b/test/mutex_test.dart @@ -1,6 +1,6 @@ import 'dart:isolate'; -import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:test/test.dart'; void main() { From 30b6b6c123ce880fb4388ab1b69d58a142b723d6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 26 Jan 2024 11:50:28 +0200 Subject: [PATCH 21/57] standard Drift is fine for this lib. Changes are on compiled worker. --- .../abstract_isolate_connection_factory.dart | 14 +++++++------- pubspec.yaml | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/lib/src/common/abstract_isolate_connection_factory.dart b/lib/src/common/abstract_isolate_connection_factory.dart index 624a7b0..9410c8a 100644 --- a/lib/src/common/abstract_isolate_connection_factory.dart +++ b/lib/src/common/abstract_isolate_connection_factory.dart @@ -1,12 +1,13 @@ import 'dart:async'; -import 'package:sqlite_async/sqlite3_common.dart'; -import 'package:sqlite_async/definitions.dart'; +import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/src/sqlite_connection.dart'; import 'abstract_open_factory.dart'; /// A connection factory that can be passed to different isolates. -abstract class AbstractIsolateConnectionFactory { - AbstractDefaultSqliteOpenFactory get openFactory; +abstract class AbstractIsolateConnectionFactory< + Database extends sqlite.CommonDatabase> { + AbstractDefaultSqliteOpenFactory get openFactory; /// Open a new SqliteConnection. /// @@ -20,9 +21,8 @@ abstract class AbstractIsolateConnectionFactory { /// with SQLITE_BUSY if another isolate is using the database at the same time. /// 2. Other connections are not notified of any updates to tables made within /// this connection. - Future openRawDatabase({bool readOnly = false}) async { - final db = await openFactory + FutureOr openRawDatabase({bool readOnly = false}) async { + return openFactory .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); - return db; } } diff --git a/pubspec.yaml b/pubspec.yaml index 8f19f10..f1381f6 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,13 +6,12 @@ environment: sdk: ">=2.19.1 <4.0.0" dependencies: - drift: - path: "/Users/stevenontong/Documents/platform_code/powersync/drift/drift" + drift: ^2.15.0 sqlite3: ">=2.3.0 <3.0.0" async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 - meta: ^1.11.0 + meta: ^1.10.0 dev_dependencies: lints: ^2.0.0 From 6fc0e10a7e280084496f52c43bded6e86b1df5d8 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 26 Jan 2024 11:54:37 +0200 Subject: [PATCH 22/57] update sqlite dependency range for test and native compatibility --- pubspec.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index f1381f6..704f99a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -7,7 +7,9 @@ environment: dependencies: drift: ^2.15.0 - sqlite3: ">=2.3.0 <3.0.0" + sqlite3: ">=1.10.1 <3.0.0" + # Note Web support works from 2.3.0 + # sqlite3: ">=2.3.0 <3.0.0" async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From c2371fc9d057d0608ce4eaf824f045a4405da821 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 26 Jan 2024 14:33:00 +0200 Subject: [PATCH 23/57] test sdk 3.2.0 --- .github/workflows/test.yaml | 20 ++++++++++---------- pubspec.yaml | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bb45171..046f812 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ name: Test on: push: branches: - - "**" + - '**' jobs: build: @@ -30,15 +30,15 @@ jobs: strategy: matrix: include: - - sqlite_version: "3420000" - sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" - dart_sdk: 3.0.6 - - sqlite_version: "3410100" - sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" - dart_sdk: 2.19.1 - - sqlite_version: "3380000" - sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" - dart_sdk: 2.19.1 + - sqlite_version: '3420000' + sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz' + dart_sdk: 3.2.0 + # - sqlite_version: '3410100' + # sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz' + # dart_sdk: 2.19.1 + # - sqlite_version: '3380000' + # sqlite_url: 'https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz' + # dart_sdk: 2.19.1 steps: - uses: actions/checkout@v3 - uses: dart-lang/setup-dart@v1 diff --git a/pubspec.yaml b/pubspec.yaml index 704f99a..d2fe98a 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,11 +3,11 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.5.2 repository: https://github.com/journeyapps/sqlite_async.dart environment: - sdk: ">=2.19.1 <4.0.0" + sdk: ">=3.2.0 <4.0.0" dependencies: drift: ^2.15.0 - sqlite3: ">=1.10.1 <3.0.0" + sqlite3: ">=2.2.0 <3.0.0" # Note Web support works from 2.3.0 # sqlite3: ">=2.3.0 <3.0.0" async: ^2.10.0 From e8b0bf84fc3322a84aa4a80dfaef8668b6e705d8 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 26 Jan 2024 14:48:04 +0200 Subject: [PATCH 24/57] lint --- lib/src/common/abstract_open_factory.dart | 1 + lib/src/common/abstract_sqlite_database.dart | 1 + lib/src/impl/stub_isolate_connection_factory.dart | 6 ++++-- lib/src/impl/stub_sqlite_database.dart | 1 + lib/src/native/database/native_sqlite_database.dart | 1 + lib/src/native/database/port_channel.dart | 2 +- lib/src/native/native_isolate_connection_factory.dart | 1 + lib/src/web/database/web_db_context.dart | 4 ++-- lib/src/web/database/web_sqlite_database.dart | 1 - lib/src/web/web_isolate_connection_factory.dart | 2 ++ pubspec.yaml | 1 + test/json1_test.dart | 1 - 12 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/src/common/abstract_open_factory.dart b/lib/src/common/abstract_open_factory.dart index 08d79d4..e3ffda0 100644 --- a/lib/src/common/abstract_open_factory.dart +++ b/lib/src/common/abstract_open_factory.dart @@ -44,6 +44,7 @@ class SqliteOpenOptions { abstract class AbstractDefaultSqliteOpenFactory< Database extends sqlite.CommonDatabase> implements SqliteOpenFactory { + @override final String path; final SqliteOptions sqliteOptions; diff --git a/lib/src/common/abstract_sqlite_database.dart b/lib/src/common/abstract_sqlite_database.dart index f0d365d..fbeaa84 100644 --- a/lib/src/common/abstract_sqlite_database.dart +++ b/lib/src/common/abstract_sqlite_database.dart @@ -26,6 +26,7 @@ abstract class AbstractSqliteDatabase extends SqliteConnection AbstractDefaultSqliteOpenFactory get openFactory; /// Use this stream to subscribe to notifications of updates to tables. + @override Stream get updates; final StreamController updatesController = diff --git a/lib/src/impl/stub_isolate_connection_factory.dart b/lib/src/impl/stub_isolate_connection_factory.dart index 7182579..7a02bc0 100644 --- a/lib/src/impl/stub_isolate_connection_factory.dart +++ b/lib/src/impl/stub_isolate_connection_factory.dart @@ -2,17 +2,18 @@ import 'dart:async'; import 'package:sqlite3/common.dart'; import 'package:sqlite_async/definitions.dart'; -import 'package:sqlite_async/src/common/abstract_open_factory.dart'; -import '../common/abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { + @override AbstractDefaultSqliteOpenFactory openFactory; IsolateConnectionFactory({ required this.openFactory, }); + @override + /// Open a new SqliteConnection. /// /// This opens a single connection in a background execution isolate. @@ -27,6 +28,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { /// with SQLITE_BUSY if another isolate is using the database at the same time. /// 2. Other connections are not notified of any updates to tables made within /// this connection. + @override Future openRawDatabase({bool readOnly = false}) async { throw UnimplementedError(); } diff --git a/lib/src/impl/stub_sqlite_database.dart b/lib/src/impl/stub_sqlite_database.dart index ca63288..fedd9f1 100644 --- a/lib/src/impl/stub_sqlite_database.dart +++ b/lib/src/impl/stub_sqlite_database.dart @@ -9,6 +9,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override bool get closed => throw UnimplementedError(); + @override AbstractDefaultSqliteOpenFactory openFactory; @override diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index ba5573e..cb7b52e 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -141,6 +141,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// A connection factory that can be passed to different isolates. /// /// Use this to access the database in background isolates. + @override IsolateConnectionFactory isolateConnectionFactory() { return IsolateConnectionFactory( openFactory: openFactory, diff --git a/lib/src/native/database/port_channel.dart b/lib/src/native/database/port_channel.dart index 3d1ac42..c32b7b9 100644 --- a/lib/src/native/database/port_channel.dart +++ b/lib/src/native/database/port_channel.dart @@ -286,7 +286,7 @@ class _PortChannelResult { return _result as T; } else { if (_error != null && stackTrace != null) { - Error.throwWithStackTrace(_error!, stackTrace!); + Error.throwWithStackTrace(_error, stackTrace!); } else { throw _error!; } diff --git a/lib/src/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart index 2e9a01e..cdff943 100644 --- a/lib/src/native/native_isolate_connection_factory.dart +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -26,6 +26,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { /// Open a new SqliteConnection. /// /// This opens a single connection in a background execution isolate. + @override SqliteConnection open({String? debugName, bool readOnly = false}) { final updates = _IsolateUpdateListener(upstreamPort); diff --git a/lib/src/web/database/web_db_context.dart b/lib/src/web/database/web_db_context.dart index 74160e3..edfaf84 100644 --- a/lib/src/web/database/web_db_context.dart +++ b/lib/src/web/database/web_db_context.dart @@ -7,7 +7,7 @@ import 'executor/sqlite_executor.dart'; class WebReadContext implements SqliteReadContext { SQLExecutor db; - WebReadContext(SQLExecutor this.db); + WebReadContext(this.db); @override Future computeWithDatabase( @@ -38,7 +38,7 @@ class WebReadContext implements SqliteReadContext { } class WebWriteContext extends WebReadContext implements SqliteWriteContext { - WebWriteContext(SQLExecutor super.db); + WebWriteContext(super.db); @override Future execute(String sql, diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index 1eec9c3..59fd4d3 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -71,7 +71,6 @@ class SqliteDatabase extends AbstractSqliteDatabase { } Future _init() async { - await _connection.isInitialized; _connection.updates.forEach((update) { updatesController.add(update); }); diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index 502b7ee..dc2cacb 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -19,6 +19,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { /// Open a new SqliteConnection. /// /// This opens a single connection in a background execution isolate. + @override WebSqliteConnectionImpl open({String? debugName, bool readOnly = false}) { return WebSqliteConnectionImpl(mutex: mutex, openFactory: openFactory); } @@ -30,6 +31,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { /// with SQLITE_BUSY if another isolate is using the database at the same time. /// 2. Other connections are not notified of any updates to tables made within /// this connection. + @override Future openRawDatabase({bool readOnly = false}) async { return openFactory .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); diff --git a/pubspec.yaml b/pubspec.yaml index d2fe98a..f3c074d 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -10,6 +10,7 @@ dependencies: sqlite3: ">=2.2.0 <3.0.0" # Note Web support works from 2.3.0 # sqlite3: ">=2.3.0 <3.0.0" + js: ^0.7.0 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 diff --git a/test/json1_test.dart b/test/json1_test.dart index 8443f5b..a9d29e2 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -1,5 +1,4 @@ import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; import 'package:test/test.dart'; import 'util.dart'; From 699f3a3e1fdae05723fd88f0c314c08176c5bccb Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 26 Jan 2024 15:02:09 +0200 Subject: [PATCH 25/57] js versions --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index f3c074d..c3ab38f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -10,7 +10,7 @@ dependencies: sqlite3: ">=2.2.0 <3.0.0" # Note Web support works from 2.3.0 # sqlite3: ">=2.3.0 <3.0.0" - js: ^0.7.0 + js: ^0.6.3 async: ^2.10.0 collection: ^1.17.0 mutex: ^3.1.0 From c2d3e724cff829d5e6955c75609fdebdb2268745 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 29 Jan 2024 09:01:22 +0200 Subject: [PATCH 26/57] update from main --- lib/src/impl/stub_sqlite_database.dart | 6 ++++++ lib/src/native/database/native_sqlite_database.dart | 7 +++++++ lib/src/web/database/web_db_context.dart | 8 ++++++++ lib/src/web/database/web_sqlite_connection_impl.dart | 5 +++++ lib/src/web/database/web_sqlite_database.dart | 5 +++++ 5 files changed, 31 insertions(+) diff --git a/lib/src/impl/stub_sqlite_database.dart b/lib/src/impl/stub_sqlite_database.dart index fedd9f1..5f65870 100644 --- a/lib/src/impl/stub_sqlite_database.dart +++ b/lib/src/impl/stub_sqlite_database.dart @@ -56,4 +56,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { // TODO: implement isolateConnectionFactory throw UnimplementedError(); } + + @override + Future getAutoCommit() { + // TODO: implement getAutoCommit + throw UnimplementedError(); + } } diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index cb7b52e..db61296 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -94,6 +94,13 @@ class SqliteDatabase extends AbstractSqliteDatabase { return _pool.closed; } + /// Returns true if the _write_ connection is in auto-commit mode + /// (no active transaction). + @override + Future getAutoCommit() { + return _pool.getAutoCommit(); + } + void _listenForEvents() { UpdateNotification? updates; diff --git a/lib/src/web/database/web_db_context.dart b/lib/src/web/database/web_db_context.dart index edfaf84..b2d563e 100644 --- a/lib/src/web/database/web_db_context.dart +++ b/lib/src/web/database/web_db_context.dart @@ -35,6 +35,14 @@ class WebReadContext implements SqliteReadContext { return null; } } + + @override + bool get closed => throw UnimplementedError(); + + @override + Future getAutoCommit() { + throw UnimplementedError(); + } } class WebWriteContext extends WebReadContext implements SqliteWriteContext { diff --git a/lib/src/web/database/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart index 6d93f1f..0538944 100644 --- a/lib/src/web/database/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -66,4 +66,9 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { await isInitialized; await executor!.close(); } + + @override + Future getAutoCommit() { + throw UnimplementedError(); + } } diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index 59fd4d3..99f33a0 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -99,4 +99,9 @@ class SqliteDatabase extends AbstractSqliteDatabase { IsolateConnectionFactory isolateConnectionFactory() { return _isolateConnectionFactory; } + + @override + Future getAutoCommit() { + throw UnimplementedError(); + } } From 128357328ee35a08ffbc2a4afa4dc35be712f4e6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 29 Jan 2024 09:42:12 +0200 Subject: [PATCH 27/57] added comments --- README.md | 21 ++++++++++++++++++- lib/src/impl/stub_sqlite_database.dart | 3 --- lib/src/web/database/web_locks.dart | 2 +- .../web/web_isolate_connection_factory.dart | 2 ++ lib/src/web/web_sqlite_open_factory.dart | 1 + pubspec.yaml | 6 ++---- test/what.dart | 7 +++++++ 7 files changed, 33 insertions(+), 9 deletions(-) create mode 100644 test/what.dart diff --git a/README.md b/README.md index 1f320c8..38552e4 100644 --- a/README.md +++ b/README.md @@ -77,4 +77,23 @@ void main() async { ``` # Web -Web requires sqlite3.dart version 2.3.0 or greater with the matching WASM file provided. \ No newline at end of file + +Web support is provided by the [Drift](https://drift.simonbinder.eu/web/) library. + +Web support requires Sqlite3 WASM and Drift worker Javascript files to be accessible via configurable URIs. + +Default URIs are shown in the example below. URIs only need to be specified if they differ from default values. + +Setup + +``` Dart +import 'package:sqlite_async/sqlite_async.dart'; + +final db = SqliteDatabase( + path: 'test', + options: SqliteOptions( + webSqliteOptions: WebSqliteOptions( + wasmUri: 'sqlite3.wasm', workerUri: 'drift_worker.js'))); + +``` + diff --git a/lib/src/impl/stub_sqlite_database.dart b/lib/src/impl/stub_sqlite_database.dart index 5f65870..bb317f9 100644 --- a/lib/src/impl/stub_sqlite_database.dart +++ b/lib/src/impl/stub_sqlite_database.dart @@ -47,19 +47,16 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override Future close() { - // TODO: implement close throw UnimplementedError(); } @override AbstractIsolateConnectionFactory isolateConnectionFactory() { - // TODO: implement isolateConnectionFactory throw UnimplementedError(); } @override Future getAutoCommit() { - // TODO: implement getAutoCommit throw UnimplementedError(); } } diff --git a/lib/src/web/database/web_locks.dart b/lib/src/web/database/web_locks.dart index 7963820..a54af54 100644 --- a/lib/src/web/database/web_locks.dart +++ b/lib/src/web/database/web_locks.dart @@ -6,7 +6,7 @@ import 'package:js/js.dart'; @JS('navigator.locks') external NavigatorLocks navigatorLocks; +/// TODO Web navigator lock interface should be used to support multiple tabs abstract class NavigatorLocks { Future request(String name, Function callbacks); - // Future request(String name, Future Function(dynamic lock) callback); } diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index dc2cacb..b0ca9fc 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -25,6 +25,8 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { } /// Opens a synchronous sqlite.Database directly in the current isolate. + /// This should not be used in conjunction with async connections provided + /// by Drift. /// /// This gives direct access to the database, but: /// 1. No app-level locking is performed automatically. Transactions may fail diff --git a/lib/src/web/web_sqlite_open_factory.dart b/lib/src/web/web_sqlite_open_factory.dart index 50f9df9..fa73071 100644 --- a/lib/src/web/web_sqlite_open_factory.dart +++ b/lib/src/web/web_sqlite_open_factory.dart @@ -19,6 +19,7 @@ class DefaultSqliteOpenFactory /// It is possible to open a CommonDatabase in the main Dart/JS context with standard sqlite3.dart, /// This connection requires an external Webworker implementation for asynchronous operations. + /// Do not use this in conjunction with async connections provided by Drift Future openDB(SqliteOpenOptions options) async { final wasmSqlite = await WasmSqlite3.loadFromUrl( Uri.parse(sqliteOptions.webSqliteOptions.wasmUri)); diff --git a/pubspec.yaml b/pubspec.yaml index 7952919..c8a99fc 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -3,13 +3,11 @@ description: High-performance asynchronous interface for SQLite on Dart and Flut version: 0.6.0 repository: https://github.com/powersync-ja/sqlite_async.dart environment: - sdk: ">=3.2.0 <4.0.0" + sdk: '>=3.2.0 <4.0.0' dependencies: drift: ^2.15.0 - sqlite3: "^2.3.0" - # Note Web support works from 2.3.0 - # sqlite3: ">=2.3.0 <3.0.0" + sqlite3: '^2.3.0' js: ^0.6.3 async: ^2.10.0 collection: ^1.17.0 diff --git a/test/what.dart b/test/what.dart new file mode 100644 index 0000000..5a3ebdc --- /dev/null +++ b/test/what.dart @@ -0,0 +1,7 @@ +import 'package:sqlite_async/sqlite_async.dart'; + +final db = SqliteDatabase( + path: 'test', + options: SqliteOptions( + webSqliteOptions: WebSqliteOptions( + wasmUri: 'sqlite3.wasm', workerUri: 'drift.worker.js'))); From 852633b5d5f06b37c3cf2ca59db1ede14458cc40 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 29 Jan 2024 09:47:07 +0200 Subject: [PATCH 28/57] added readme note --- .github/workflows/test.yaml | 22 +++++++++++----------- README.md | 2 ++ test/what.dart | 7 ------- 3 files changed, 13 insertions(+), 18 deletions(-) delete mode 100644 test/what.dart diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b22f322..c9fa2df 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ name: Test on: push: branches: - - '**' + - "**" jobs: build: @@ -30,20 +30,20 @@ jobs: strategy: matrix: include: - - sqlite_version: '3440200' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz' + - sqlite_version: "3440200" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3430200' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz' + - sqlite_version: "3430200" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3420000' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz' + - sqlite_version: "3420000" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3410100' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz' + - sqlite_version: "3410100" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3380000' - sqlite_url: 'https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz' + - sqlite_version: "3380000" + sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" dart_sdk: 3.2.0 steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 38552e4..890fa64 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,8 @@ Web support requires Sqlite3 WASM and Drift worker Javascript files to be access Default URIs are shown in the example below. URIs only need to be specified if they differ from default values. +Watched queries and table change notifications are only supported when using a custom Drift worker. [TBD release link] + Setup ``` Dart diff --git a/test/what.dart b/test/what.dart deleted file mode 100644 index 5a3ebdc..0000000 --- a/test/what.dart +++ /dev/null @@ -1,7 +0,0 @@ -import 'package:sqlite_async/sqlite_async.dart'; - -final db = SqliteDatabase( - path: 'test', - options: SqliteOptions( - webSqliteOptions: WebSqliteOptions( - wasmUri: 'sqlite3.wasm', workerUri: 'drift.worker.js'))); From eff0a10fe08b2f19abed590dce81575f9ac9b1bb Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 29 Jan 2024 16:42:24 +0200 Subject: [PATCH 29/57] use mutexes more in abstractions. --- lib/sqlite_async.dart | 1 + .../abstract_isolate_connection_factory.dart | 6 + lib/src/common/abstract_mutex.dart | 6 + .../connection/sync_sqlite_connection.dart | 118 ++++++++++++++++++ .../database => common}/port_channel.dart | 2 +- .../impl/stub_isolate_connection_factory.dart | 8 ++ lib/src/native/database/connection_pool.dart | 2 +- .../native_sqlite_connection_impl.dart | 2 +- .../database/native_sqlite_database.dart | 2 +- .../native_isolate_connection_factory.dart | 5 +- lib/src/native/native_isolate_mutex.dart | 26 +++- .../web/web_isolate_connection_factory.dart | 4 + 12 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 lib/src/common/connection/sync_sqlite_connection.dart rename lib/src/{native/database => common}/port_channel.dart (99%) diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index a354c3a..ec5f2bc 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -12,3 +12,4 @@ export 'src/sqlite_queries.dart'; export 'src/update_notification.dart'; export 'src/utils.dart'; export 'definitions.dart'; +export 'src/common/connection/sync_sqlite_connection.dart'; diff --git a/lib/src/common/abstract_isolate_connection_factory.dart b/lib/src/common/abstract_isolate_connection_factory.dart index 9410c8a..c3903e5 100644 --- a/lib/src/common/abstract_isolate_connection_factory.dart +++ b/lib/src/common/abstract_isolate_connection_factory.dart @@ -1,14 +1,20 @@ import 'dart:async'; +import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite3_common.dart' as sqlite; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'abstract_open_factory.dart'; +import 'port_channel.dart'; /// A connection factory that can be passed to different isolates. abstract class AbstractIsolateConnectionFactory< Database extends sqlite.CommonDatabase> { AbstractDefaultSqliteOpenFactory get openFactory; + AbstractMutex get mutex; + + SerializedPortClient get upstreamPort; + /// Open a new SqliteConnection. /// /// This opens a single connection in a background execution isolate. diff --git a/lib/src/common/abstract_mutex.dart b/lib/src/common/abstract_mutex.dart index becd581..3878991 100644 --- a/lib/src/common/abstract_mutex.dart +++ b/lib/src/common/abstract_mutex.dart @@ -2,6 +2,12 @@ abstract class AbstractMutex { /// timeout is a timeout for acquiring the lock, not for the callback Future lock(Future Function() callback, {Duration? timeout}); + /// Use [open] to get a [AbstractMutex] instance. + /// This is mainly used for shared mutexes + AbstractMutex open() { + return this; + } + /// Release resources used by the Mutex. /// /// Subsequent calls to [lock] may fail, or may never call the callback. diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart new file mode 100644 index 0000000..da0623f --- /dev/null +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -0,0 +1,118 @@ +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/src/mutex.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; + +/// A simple "synchronous" connection which provides the SqliteConnection +/// implementation using a synchronous connection +class SyncSqliteConnection extends SqliteConnection with SqliteQueries { + final CommonDatabase db; + AbstractMutex mutex; + @override + late final Stream updates; + + bool _closed = false; + + SyncSqliteConnection(this.db, this.mutex) { + updates = db.updates.map( + (event) { + return UpdateNotification(Set.from([event.tableName])); + }, + ); + } + + @override + Future readLock(Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return mutex.lock(() => callback(SyncReadContext(db)), + timeout: lockTimeout); + } + + @override + Future writeLock(Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, String? debugContext}) { + return mutex.lock(() => callback(SyncWriteContext(db)), + timeout: lockTimeout); + } + + @override + Future close() async { + _closed = true; + return db.dispose(); + } + + @override + bool get closed => _closed; + + @override + Future getAutoCommit() async { + return db.autocommit; + } +} + +class SyncReadContext implements SqliteReadContext { + CommonDatabase db; + + SyncReadContext(this.db); + + @override + Future computeWithDatabase( + Future Function(CommonDatabase db) compute) { + return compute(db); + } + + @override + Future get(String sql, [List parameters = const []]) async { + return db.select(sql, parameters).first; + } + + @override + Future getAll(String sql, + [List parameters = const []]) async { + return db.select(sql, parameters); + } + + @override + Future getOptional(String sql, + [List parameters = const []]) async { + try { + return await db.select(sql, parameters).first; + } catch (ex) { + return null; + } + } + + @override + bool get closed => false; + + @override + Future getAutoCommit() async { + return db.autocommit; + } +} + +class SyncWriteContext extends SyncReadContext implements SqliteWriteContext { + SyncWriteContext(super.db); + + @override + Future execute(String sql, + [List parameters = const []]) async { + return db.select(sql, parameters); + } + + @override + Future executeBatch( + String sql, List> parameterSets) async { + return computeWithDatabase((db) async { + final statement = db.prepare(sql, checkNoTail: true); + try { + for (var parameters in parameterSets) { + statement.execute(parameters); + } + } finally { + statement.dispose(); + } + }); + } +} diff --git a/lib/src/native/database/port_channel.dart b/lib/src/common/port_channel.dart similarity index 99% rename from lib/src/native/database/port_channel.dart rename to lib/src/common/port_channel.dart index c32b7b9..3d1ac42 100644 --- a/lib/src/native/database/port_channel.dart +++ b/lib/src/common/port_channel.dart @@ -286,7 +286,7 @@ class _PortChannelResult { return _result as T; } else { if (_error != null && stackTrace != null) { - Error.throwWithStackTrace(_error, stackTrace!); + Error.throwWithStackTrace(_error!, stackTrace!); } else { throw _error!; } diff --git a/lib/src/impl/stub_isolate_connection_factory.dart b/lib/src/impl/stub_isolate_connection_factory.dart index 7a02bc0..ffc6416 100644 --- a/lib/src/impl/stub_isolate_connection_factory.dart +++ b/lib/src/impl/stub_isolate_connection_factory.dart @@ -2,6 +2,8 @@ import 'dart:async'; import 'package:sqlite3/common.dart'; import 'package:sqlite_async/definitions.dart'; +import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @@ -32,4 +34,10 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { Future openRawDatabase({bool readOnly = false}) async { throw UnimplementedError(); } + + @override + AbstractMutex get mutex => throw UnimplementedError(); + + @override + SerializedPortClient get upstreamPort => throw UnimplementedError(); } diff --git a/lib/src/native/database/connection_pool.dart b/lib/src/native/database/connection_pool.dart index 74a89ac..7777d1b 100644 --- a/lib/src/native/database/connection_pool.dart +++ b/lib/src/native/database/connection_pool.dart @@ -6,7 +6,7 @@ import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import '../../sqlite_connection.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; -import 'port_channel.dart'; +import '../../common/port_channel.dart'; import 'native_sqlite_connection_impl.dart'; import '../native_sqlite_open_factory.dart'; diff --git a/lib/src/native/database/native_sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart index 6f5886b..5f18a21 100644 --- a/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -8,7 +8,7 @@ import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import '../../utils/database_utils.dart'; -import 'port_channel.dart'; +import '../../common/port_channel.dart'; import '../../sqlite_connection.dart'; import '../../sqlite_queries.dart'; import '../../update_notification.dart'; diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index db61296..60cddf9 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -10,7 +10,7 @@ import '../native_isolate_connection_factory.dart'; import '../../sqlite_options.dart'; import '../../update_notification.dart'; import '../../common/abstract_sqlite_database.dart'; -import 'port_channel.dart'; +import '../../common/port_channel.dart'; import 'connection_pool.dart'; import 'native_sqlite_connection_impl.dart'; diff --git a/lib/src/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart index cdff943..d93b125 100644 --- a/lib/src/native/native_isolate_connection_factory.dart +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -6,7 +6,7 @@ import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import '../sqlite_connection.dart'; import '../update_notification.dart'; import '../utils/native_database_utils.dart'; -import 'database/port_channel.dart'; +import '../common/port_channel.dart'; import 'database/native_sqlite_connection_impl.dart'; import '../common/abstract_isolate_connection_factory.dart'; @@ -15,7 +15,10 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @override DefaultSqliteOpenFactory openFactory; + @override SerializedMutex mutex; + + @override SerializedPortClient upstreamPort; IsolateConnectionFactory( diff --git a/lib/src/native/native_isolate_mutex.dart b/lib/src/native/native_isolate_mutex.dart index a252a67..2d28b17 100644 --- a/lib/src/native/native_isolate_mutex.dart +++ b/lib/src/native/native_isolate_mutex.dart @@ -4,7 +4,7 @@ import 'dart:async'; import 'package:sqlite_async/src/common/abstract_mutex.dart'; -import 'package:sqlite_async/src/native/database/port_channel.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; abstract class Mutex extends AbstractMutex { factory Mutex() { @@ -87,6 +87,11 @@ class SimpleMutex implements Mutex { }); } + @override + open() { + return this; + } + @override Future close() async { _shared?.close(); @@ -104,14 +109,24 @@ class SimpleMutex implements Mutex { /// Use [open] to get a [SharedMutex] instance. /// /// Uses a [SendPort] to communicate with the source mutex. -class SerializedMutex { +class SerializedMutex extends AbstractMutex { final SerializedPortClient client; - const SerializedMutex(this.client); + SerializedMutex(this.client); SharedMutex open() { return SharedMutex._(client.open()); } + + @override + Future close() { + throw UnimplementedError(); + } + + @override + Future lock(Future Function() callback, {Duration? timeout}) { + throw UnimplementedError(); + } } /// Mutex instantiated from a source mutex, potentially in a different isolate. @@ -146,6 +161,11 @@ class SharedMutex implements Mutex { client.fire(const _UnlockMessage()); } + @override + open() { + return this; + } + Future _acquire({Duration? timeout}) async { final lockFuture = client.post(const _AcquireMessage()); bool timedout = false; diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index b0ca9fc..bcae91f 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -3,6 +3,7 @@ import 'dart:async'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'web_mutex.dart'; import 'database/web_sqlite_connection_impl.dart'; @@ -38,4 +39,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { return openFactory .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); } + + @override + SerializedPortClient get upstreamPort => throw UnimplementedError(); } From 9d3786a6580f8b0b92c37c48f64bd6ea7a6ca84a Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 11:42:46 +0200 Subject: [PATCH 30/57] wip: abstract tests. Add Web server for WASM files --- .gitignore | 3 + lib/src/common/port_channel.dart | 2 +- pubspec.yaml | 4 ++ scripts/benchmark.dart | 6 +- test/basic_test.dart | 45 +++++++------ test/close_test.dart | 13 ++-- test/isolate_test.dart | 13 ++-- test/json1_test.dart | 12 ++-- test/migration_test.dart | 18 +++--- test/mutex_test.dart | 1 + test/server/asset_server.dart | 41 ++++++++++++ test/server/worker_server.dart | 47 ++++++++++++++ test/util.dart | 99 ----------------------------- test/utils/abstract_test_utils.dart | 43 +++++++++++++ test/utils/native_test_utils.dart | 98 ++++++++++++++++++++++++++++ test/utils/stub_test_utils.dart | 18 ++++++ test/utils/test_utils_impl.dart | 5 ++ test/utils/web_test_utils.dart | 53 +++++++++++++++ test/watch_test.dart | 23 ++++--- 19 files changed, 388 insertions(+), 156 deletions(-) create mode 100644 test/server/asset_server.dart create mode 100644 test/server/worker_server.dart delete mode 100644 test/util.dart create mode 100644 test/utils/abstract_test_utils.dart create mode 100644 test/utils/native_test_utils.dart create mode 100644 test/utils/stub_test_utils.dart create mode 100644 test/utils/test_utils_impl.dart create mode 100644 test/utils/web_test_utils.dart diff --git a/.gitignore b/.gitignore index 0a5c131..5c41ae2 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ # https://dart.dev/guides/libraries/private-files#pubspeclock. pubspec.lock +# Test assets +assets + .idea .vscode *.db diff --git a/lib/src/common/port_channel.dart b/lib/src/common/port_channel.dart index 3d1ac42..c32b7b9 100644 --- a/lib/src/common/port_channel.dart +++ b/lib/src/common/port_channel.dart @@ -286,7 +286,7 @@ class _PortChannelResult { return _result as T; } else { if (_error != null && stackTrace != null) { - Error.throwWithStackTrace(_error!, stackTrace!); + Error.throwWithStackTrace(_error, stackTrace!); } else { throw _error!; } diff --git a/pubspec.yaml b/pubspec.yaml index c8a99fc..c065f8d 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -20,3 +20,7 @@ dev_dependencies: test_api: ^0.7.0 glob: ^2.1.1 benchmarking: ^0.6.1 + shelf: ^1.4.1 + shelf_static: ^1.1.2 + stream_channel: ^2.1.2 + path: ^1.9.0 diff --git a/scripts/benchmark.dart b/scripts/benchmark.dart index a93fea1..c420e5f 100644 --- a/scripts/benchmark.dart +++ b/scripts/benchmark.dart @@ -6,7 +6,9 @@ import 'package:benchmarking/benchmarking.dart'; import 'package:collection/collection.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import '../test/util.dart'; +import '../test/utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); typedef BenchmarkFunction = Future Function( SqliteDatabase, List>); @@ -176,7 +178,7 @@ void main() async { await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); } - final db = await setupDatabase(path: 'test-db/benchmark.db'); + final db = await testUtils.setupDatabase(path: 'test-db/benchmark.db'); await db.execute('PRAGMA wal_autocheckpoint = 0'); await createTables(db); diff --git a/test/basic_test.dart b/test/basic_test.dart index 9b563c8..6b0f279 100644 --- a/test/basic_test.dart +++ b/test/basic_test.dart @@ -1,24 +1,27 @@ import 'dart:async'; import 'dart:math'; -import 'package:sqlite3/sqlite3.dart' as sqlite; +import 'package:sqlite3/common.dart' as sqlite; import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Basic Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.init(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); createTables(SqliteDatabase db) async { @@ -29,7 +32,7 @@ void main() { } test('Basic Setup', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.execute( @@ -50,8 +53,8 @@ void main() { // Manually verified test('Concurrency', () async { - final db = - SqliteDatabase.withFactory(testFactory(path: path), maxReaders: 3); + final db = SqliteDatabase.withFactory(testUtils.testFactory(path: path), + maxReaders: 3); await db.initialize(); await createTables(db); @@ -65,7 +68,7 @@ void main() { }); test('read-only transactions', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Can read @@ -104,7 +107,7 @@ void main() { test('should not allow direct db calls within a transaction callback', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.writeTransaction((tx) async { @@ -117,7 +120,7 @@ void main() { test('should not allow read-only db calls within transaction callback', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.writeTransaction((tx) async { @@ -141,7 +144,7 @@ void main() { }); test('should not allow read-only db calls within lock callback', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Locks - should behave the same as transactions above @@ -165,7 +168,7 @@ void main() { test( 'should allow read-only db calls within transaction callback in separate zone', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Get a reference to the parent zone (outside the transaction). @@ -202,7 +205,7 @@ void main() { }); test('should allow PRAMGAs', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); // Not allowed in transactions, but does work as a direct statement. await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); @@ -210,7 +213,7 @@ void main() { }); test('should allow ignoring errors', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); ignore(db.execute( @@ -218,7 +221,7 @@ void main() { }); test('should properly report errors in transactions', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); var tp = db.writeTransaction((tx) async { @@ -262,7 +265,7 @@ void main() { }); test('should error on dangling transactions', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await expectLater(() async { await db.execute('BEGIN'); @@ -270,7 +273,7 @@ void main() { }); test('should handle normal errors', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); Error? caughtError; final syntheticError = ArgumentError('foobar'); @@ -289,7 +292,7 @@ void main() { }); test('should handle uncaught errors', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); Object? caughtError; await db.computeWithDatabase((db) async { @@ -312,7 +315,7 @@ void main() { }); test('should handle uncaught errors in read connections', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); for (var i = 0; i < 10; i++) { Object? caughtError; @@ -342,7 +345,7 @@ void main() { }); test('should allow resuming transaction after errors', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); SqliteWriteContext? savedTx; await db.writeTransaction((tx) async { diff --git a/test/close_test.dart b/test/close_test.dart index e6a4d74..d3c62dd 100644 --- a/test/close_test.dart +++ b/test/close_test.dart @@ -1,22 +1,25 @@ +@TestOn('!browser') import 'dart:io'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Close Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); createTables(AbstractSqliteDatabase db) async { @@ -31,7 +34,7 @@ void main() { // If the write connection is closed before the read connections, that is // not the case. - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); await db.execute( diff --git a/test/isolate_test.dart b/test/isolate_test.dart index 69b5035..60bea87 100644 --- a/test/isolate_test.dart +++ b/test/isolate_test.dart @@ -1,24 +1,27 @@ +@TestOn('!browser') import 'dart:isolate'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Isolate Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); test('Basic Isolate usage', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final factory = db.isolateConnectionFactory(); final result = await Isolate.run(() async { diff --git a/test/json1_test.dart b/test/json1_test.dart index a9d29e2..90b91d6 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -1,7 +1,9 @@ import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); class TestUser { int? id; @@ -24,12 +26,12 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); createTables(AbstractSqliteDatabase db) async { @@ -40,7 +42,7 @@ void main() { } test('Inserts', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); var users1 = [ diff --git a/test/migration_test.dart b/test/migration_test.dart index b0c6145..37d4c79 100644 --- a/test/migration_test.dart +++ b/test/migration_test.dart @@ -1,23 +1,25 @@ import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; -import 'util.dart'; +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { group('Basic Tests', () { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); tearDown(() async { - await cleanDb(path: path); + await testUtils.cleanDb(path: path); }); test('Basic Migrations', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( @@ -51,7 +53,7 @@ void main() { }); test('Migration with createDatabase', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( @@ -83,7 +85,7 @@ void main() { }); test('Migration with down migrations', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( @@ -135,7 +137,7 @@ void main() { }); test('Migration with double down migrations', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); final migrations = SqliteMigrations(); migrations.add(SqliteMigration(1, (tx) async { await tx.execute( diff --git a/test/mutex_test.dart b/test/mutex_test.dart index 77561e5..2e6a3ee 100644 --- a/test/mutex_test.dart +++ b/test/mutex_test.dart @@ -1,3 +1,4 @@ +@TestOn('!browser') import 'dart:isolate'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; diff --git a/test/server/asset_server.dart b/test/server/asset_server.dart new file mode 100644 index 0000000..272e0bf --- /dev/null +++ b/test/server/asset_server.dart @@ -0,0 +1,41 @@ +import 'dart:io'; + +import 'package:shelf/shelf.dart'; +import 'package:shelf/shelf_io.dart' as io; +import 'package:shelf_static/shelf_static.dart'; +import 'package:stream_channel/stream_channel.dart'; + +const _corsHeaders = {'Access-Control-Allow-Origin': '*'}; + +Middleware cors() { + Response? handleOptionsRequest(Request request) { + if (request.method == 'OPTIONS') { + return Response.ok(null, headers: _corsHeaders); + } else { + // Returning null will run the regular request handler + return null; + } + } + + Response addCorsHeaders(Response response) { + return response.change(headers: _corsHeaders); + } + + return createMiddleware( + requestHandler: handleOptionsRequest, responseHandler: addCorsHeaders); +} + +Future hybridMain(StreamChannel channel) async { + final server = await HttpServer.bind('localhost', 0); + + final handler = const Pipeline() + .addMiddleware(cors()) + .addHandler(createStaticHandler('./assets')); + io.serveRequests(server, handler); + + channel.sink.add(server.port); + await channel.stream + .listen(null) + .asFuture() + .then((_) => server.close()); +} diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart new file mode 100644 index 0000000..f581005 --- /dev/null +++ b/test/server/worker_server.dart @@ -0,0 +1,47 @@ +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:shelf/shelf.dart'; +import 'package:shelf/shelf_io.dart' as io; +import 'package:shelf_static/shelf_static.dart'; +import 'package:stream_channel/stream_channel.dart'; +import 'package:test/test.dart'; + +import 'asset_server.dart'; + +Future hybridMain(StreamChannel channel) async { + final directory = Directory.systemTemp + .createTempSync('sqlite_dart_web') + .resolveSymbolicLinksSync(); + + // Copy sqlite3.wasm file expected by the worker + await File('example/web/sqlite3.wasm') + .copy(p.join(directory, 'sqlite3.wasm')); + + // And compile worker code + final process = await Process.run(Platform.executable, [ + 'compile', + 'js', + '-o', + p.join(directory, 'worker.dart.js'), + '-O0', + 'test/wasm/worker.dart', + ]); + + if (process.exitCode != 0) { + fail('Could not compile worker'); + } + + final server = await HttpServer.bind('localhost', 0); + + final handler = const Pipeline() + .addMiddleware(cors()) + .addHandler(createStaticHandler(directory)); + io.serveRequests(server, handler); + + channel.sink.add(server.port); + await channel.stream.listen(null).asFuture().then((_) async { + await server.close(); + await Directory(directory).delete(); + }); +} diff --git a/test/util.dart b/test/util.dart deleted file mode 100644 index f14ae9e..0000000 --- a/test/util.dart +++ /dev/null @@ -1,99 +0,0 @@ -import 'dart:async'; -import 'dart:ffi'; -import 'dart:io'; -import 'dart:isolate'; - -import 'package:glob/glob.dart'; -import 'package:glob/list_local_fs.dart'; -import 'package:sqlite3/common.dart'; -import 'package:sqlite3/open.dart' as sqlite_open; -import 'package:sqlite3/sqlite3.dart' as sqlite; -import 'package:sqlite_async/sqlite_async.dart'; -import 'package:test_api/src/backend/invoker.dart'; - -const defaultSqlitePath = 'libsqlite3.so.0'; -// const defaultSqlitePath = './sqlite-autoconf-3410100/.libs/libsqlite3.so.0'; - -class TestSqliteOpenFactory extends DefaultSqliteOpenFactory { - String sqlitePath; - - TestSqliteOpenFactory( - {required super.path, - super.sqliteOptions, - this.sqlitePath = defaultSqlitePath}); - - @override - FutureOr open(SqliteOpenOptions options) async { - sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { - return DynamicLibrary.open(sqlitePath); - }); - final db = await super.open(options); - - db.createFunction( - functionName: 'test_sleep', - argumentCount: const sqlite.AllowedArgumentCount(1), - function: (args) { - final millis = args[0] as int; - sleep(Duration(milliseconds: millis)); - return millis; - }, - ); - - db.createFunction( - functionName: 'test_connection_name', - argumentCount: const sqlite.AllowedArgumentCount(0), - function: (args) { - return Isolate.current.debugName; - }, - ); - - return db; - } -} - -DefaultSqliteOpenFactory testFactory({String? path}) { - return TestSqliteOpenFactory(path: path ?? dbPath()); -} - -Future setupDatabase({String? path}) async { - final db = SqliteDatabase.withFactory(testFactory(path: path)); - await db.initialize(); - return db; -} - -Future cleanDb({required String path}) async { - try { - await File(path).delete(); - } on PathNotFoundException { - // Not an issue - } - try { - await File("$path-shm").delete(); - } on PathNotFoundException { - // Not an issue - } - try { - await File("$path-wal").delete(); - } on PathNotFoundException { - // Not an issue - } -} - -List findSqliteLibraries() { - var glob = Glob('sqlite-*/.libs/libsqlite3.so'); - List sqlites = [ - 'libsqlite3.so.0', - for (var sqlite in glob.listSync()) sqlite.path - ]; - return sqlites; -} - -String dbPath() { - final test = Invoker.current!.liveTest; - var testName = test.test.name; - var testShortName = - testName.replaceAll(RegExp(r'[\s\./]'), '_').toLowerCase(); - var dbName = "test-db/$testShortName.db"; - Directory("test-db").createSync(recursive: false); - return dbName; -} diff --git a/test/utils/abstract_test_utils.dart b/test/utils/abstract_test_utils.dart new file mode 100644 index 0000000..30f9615 --- /dev/null +++ b/test/utils/abstract_test_utils.dart @@ -0,0 +1,43 @@ +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test_api/src/backend/invoker.dart'; + +class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { + final String? sqlitePath; + + TestDefaultSqliteOpenFactory( + {required super.path, super.sqliteOptions, this.sqlitePath = ''}); +} + +abstract class AbstractTestUtils { + String dbPath() { + final test = Invoker.current!.liveTest; + var testName = test.test.name; + var testShortName = + testName.replaceAll(RegExp(r'[\s\./]'), '_').toLowerCase(); + var dbName = "test-db/$testShortName.db"; + return dbName; + } + + /// Generates a test open factory + TestDefaultSqliteOpenFactory testFactory( + {String? path, + String? sqlitePath, + SqliteOptions options = const SqliteOptions.defaults()}) { + return TestDefaultSqliteOpenFactory( + path: path ?? dbPath(), sqliteOptions: options); + } + + /// Creates a SqliteDatabaseConnection + Future setupDatabase({String? path}) async { + final db = SqliteDatabase.withFactory(testFactory(path: path)); + await db.initialize(); + return db; + } + + Future init(); + + /// Deletes any DB data + Future cleanDb({required String path}); + + List findSqliteLibraries(); +} diff --git a/test/utils/native_test_utils.dart b/test/utils/native_test_utils.dart new file mode 100644 index 0000000..faa2e11 --- /dev/null +++ b/test/utils/native_test_utils.dart @@ -0,0 +1,98 @@ +import 'dart:async'; +import 'dart:ffi'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:glob/glob.dart'; +import 'package:glob/list_local_fs.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite3/open.dart' as sqlite_open; + +import 'abstract_test_utils.dart'; + +const defaultSqlitePath = 'libsqlite3.so.0'; + +class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { + TestSqliteOpenFactory( + {required super.path, + super.sqliteOptions, + super.sqlitePath = defaultSqlitePath}); + + @override + FutureOr open(SqliteOpenOptions options) async { + sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { + return DynamicLibrary.open(sqlitePath!); + }); + final db = await super.open(options); + + db.createFunction( + functionName: 'test_sleep', + argumentCount: const AllowedArgumentCount(1), + function: (args) { + final millis = args[0] as int; + sleep(Duration(milliseconds: millis)); + return millis; + }, + ); + + db.createFunction( + functionName: 'test_connection_name', + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return Isolate.current.debugName; + }, + ); + + return db; + } +} + +class TestUtils extends AbstractTestUtils { + @override + Future init() async {} + + @override + String dbPath() { + Directory("test-db").createSync(recursive: false); + return super.dbPath(); + } + + @override + Future cleanDb({required String path}) async { + try { + await File(path).delete(); + } on PathNotFoundException { + // Not an issue + } + try { + await File("$path-shm").delete(); + } on PathNotFoundException { + // Not an issue + } + try { + await File("$path-wal").delete(); + } on PathNotFoundException { + // Not an issue + } + } + + @override + List findSqliteLibraries() { + var glob = Glob('sqlite-*/.libs/libsqlite3.so'); + List sqlites = [ + 'libsqlite3.so.0', + for (var sqlite in glob.listSync()) sqlite.path + ]; + return sqlites; + } + + @override + TestDefaultSqliteOpenFactory testFactory( + {String? path, + String? sqlitePath, + SqliteOptions options = const SqliteOptions.defaults()}) { + return TestSqliteOpenFactory( + path: path ?? dbPath(), sqlitePath: sqlitePath, sqliteOptions: options); + } +} diff --git a/test/utils/stub_test_utils.dart b/test/utils/stub_test_utils.dart new file mode 100644 index 0000000..a9ccf55 --- /dev/null +++ b/test/utils/stub_test_utils.dart @@ -0,0 +1,18 @@ +import 'abstract_test_utils.dart'; + +class TestUtils extends AbstractTestUtils { + @override + Future cleanDb({required String path}) { + throw UnimplementedError(); + } + + @override + List findSqliteLibraries() { + throw UnimplementedError(); + } + + @override + Future init() { + throw UnimplementedError(); + } +} diff --git a/test/utils/test_utils_impl.dart b/test/utils/test_utils_impl.dart new file mode 100644 index 0000000..99a34d3 --- /dev/null +++ b/test/utils/test_utils_impl.dart @@ -0,0 +1,5 @@ +export 'stub_test_utils.dart' + // ignore: uri_does_not_exist + if (dart.library.io) 'native_test_utils.dart' + // ignore: uri_does_not_exist + if (dart.library.html) 'web_test_utils.dart'; diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart new file mode 100644 index 0000000..141863d --- /dev/null +++ b/test/utils/web_test_utils.dart @@ -0,0 +1,53 @@ +import 'dart:async'; + +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test/test.dart'; +import 'abstract_test_utils.dart'; + +class TestUtils extends AbstractTestUtils { + late final Future _isInitialized; + late SqliteOptions? webOptions = null; + + TestUtils() { + _isInitialized = init(); + } + + Future init() async { + if (webOptions != null) { + return; + } + + final channel = spawnHybridUri('/test/server/asset_server.dart'); + final port = await channel.stream.first as int; + + final sqliteWasm = Uri.parse('http://localhost:$port/sqlite3.wasm'); + final sqliteDrift = Uri.parse('http://localhost:$port/drift_worker.js'); + + print('sqlite4' + sqliteWasm.toString()); + + webOptions = SqliteOptions( + webSqliteOptions: WebSqliteOptions( + wasmUri: sqliteWasm.toString(), workerUri: sqliteDrift.toString())); + } + + @override + Future cleanDb({required String path}) async {} + + @override + TestDefaultSqliteOpenFactory testFactory( + {String? path, + String? sqlitePath, + SqliteOptions options = const SqliteOptions.defaults()}) { + return super.testFactory(path: path, options: webOptions!); + } + + @override + Future setupDatabase({String? path}) { + return super.setupDatabase(path: path); + } + + @override + List findSqliteLibraries() { + return []; + } +} diff --git a/test/watch_test.dart b/test/watch_test.dart index 668a9eb..5adbc83 100644 --- a/test/watch_test.dart +++ b/test/watch_test.dart @@ -6,7 +6,10 @@ import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:sqlite_async/src/utils/database_utils.dart'; import 'package:test/test.dart'; -import 'util.dart'; + +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); void main() { createTables(AbstractSqliteDatabase db) async { @@ -26,14 +29,14 @@ void main() { late String path; setUp(() async { - path = dbPath(); - await cleanDb(path: path); + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); }); - for (var sqlite in findSqliteLibraries()) { + for (var sqlite in testUtils.findSqliteLibraries()) { test('getSourceTables - $sqlite', () async { final db = SqliteDatabase.withFactory( - TestSqliteOpenFactory(path: path, sqlitePath: sqlite)); + testUtils.testFactory(path: path, sqlitePath: sqlite)); await db.initialize(); await createTables(db); @@ -62,7 +65,7 @@ void main() { } test('watch', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); const baseTime = 20; @@ -127,7 +130,7 @@ void main() { }); test('onChange', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); const baseTime = 20; @@ -164,7 +167,7 @@ void main() { }); test('single onChange', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); const baseTime = 20; @@ -186,7 +189,7 @@ void main() { }); test('watch in isolate', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); const baseTime = 20; @@ -246,7 +249,7 @@ void main() { }); test('watch with parameters', () async { - final db = await setupDatabase(path: path); + final db = await testUtils.setupDatabase(path: path); await createTables(db); const baseTime = 20; From 96d67b98ffe3859cce451c227fe3f790b78c1d4f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 30 Jan 2024 11:45:40 +0200 Subject: [PATCH 31/57] linting fixes --- lib/src/common/connection/sync_sqlite_connection.dart | 4 ++-- lib/src/native/native_isolate_mutex.dart | 1 + lib/src/web/web_isolate_connection_factory.dart | 1 + test/utils/web_test_utils.dart | 8 ++++---- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart index da0623f..db9eb8b 100644 --- a/lib/src/common/connection/sync_sqlite_connection.dart +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -17,7 +17,7 @@ class SyncSqliteConnection extends SqliteConnection with SqliteQueries { SyncSqliteConnection(this.db, this.mutex) { updates = db.updates.map( (event) { - return UpdateNotification(Set.from([event.tableName])); + return UpdateNotification({event.tableName}); }, ); } @@ -77,7 +77,7 @@ class SyncReadContext implements SqliteReadContext { Future getOptional(String sql, [List parameters = const []]) async { try { - return await db.select(sql, parameters).first; + return db.select(sql, parameters).first; } catch (ex) { return null; } diff --git a/lib/src/native/native_isolate_mutex.dart b/lib/src/native/native_isolate_mutex.dart index 2d28b17..11aab78 100644 --- a/lib/src/native/native_isolate_mutex.dart +++ b/lib/src/native/native_isolate_mutex.dart @@ -114,6 +114,7 @@ class SerializedMutex extends AbstractMutex { SerializedMutex(this.client); + @override SharedMutex open() { return SharedMutex._(client.open()); } diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index bcae91f..abe6bac 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -13,6 +13,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @override DefaultSqliteOpenFactory openFactory; + @override Mutex mutex; IsolateConnectionFactory({required this.openFactory, required this.mutex}); diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index 141863d..380e8c4 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -6,12 +6,13 @@ import 'abstract_test_utils.dart'; class TestUtils extends AbstractTestUtils { late final Future _isInitialized; - late SqliteOptions? webOptions = null; + late SqliteOptions? webOptions; TestUtils() { _isInitialized = init(); } + @override Future init() async { if (webOptions != null) { return; @@ -23,8 +24,6 @@ class TestUtils extends AbstractTestUtils { final sqliteWasm = Uri.parse('http://localhost:$port/sqlite3.wasm'); final sqliteDrift = Uri.parse('http://localhost:$port/drift_worker.js'); - print('sqlite4' + sqliteWasm.toString()); - webOptions = SqliteOptions( webSqliteOptions: WebSqliteOptions( wasmUri: sqliteWasm.toString(), workerUri: sqliteDrift.toString())); @@ -42,7 +41,8 @@ class TestUtils extends AbstractTestUtils { } @override - Future setupDatabase({String? path}) { + Future setupDatabase({String? path}) async { + await _isInitialized; return super.setupDatabase(path: path); } From 2892d21fbd2e6ec7cbd100b927c2d9f1d6975dad Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 31 Jan 2024 09:49:14 +0200 Subject: [PATCH 32/57] fix tests --- test/utils/abstract_test_utils.dart | 4 ++-- test/utils/native_test_utils.dart | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/utils/abstract_test_utils.dart b/test/utils/abstract_test_utils.dart index 30f9615..609d000 100644 --- a/test/utils/abstract_test_utils.dart +++ b/test/utils/abstract_test_utils.dart @@ -2,7 +2,7 @@ import 'package:sqlite_async/sqlite_async.dart'; import 'package:test_api/src/backend/invoker.dart'; class TestDefaultSqliteOpenFactory extends DefaultSqliteOpenFactory { - final String? sqlitePath; + final String sqlitePath; TestDefaultSqliteOpenFactory( {required super.path, super.sqliteOptions, this.sqlitePath = ''}); @@ -21,7 +21,7 @@ abstract class AbstractTestUtils { /// Generates a test open factory TestDefaultSqliteOpenFactory testFactory( {String? path, - String? sqlitePath, + String sqlitePath = '', SqliteOptions options = const SqliteOptions.defaults()}) { return TestDefaultSqliteOpenFactory( path: path ?? dbPath(), sqliteOptions: options); diff --git a/test/utils/native_test_utils.dart b/test/utils/native_test_utils.dart index faa2e11..bddafa0 100644 --- a/test/utils/native_test_utils.dart +++ b/test/utils/native_test_utils.dart @@ -22,7 +22,7 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { @override FutureOr open(SqliteOpenOptions options) async { sqlite_open.open.overrideFor(sqlite_open.OperatingSystem.linux, () { - return DynamicLibrary.open(sqlitePath!); + return DynamicLibrary.open(sqlitePath); }); final db = await super.open(options); @@ -90,7 +90,7 @@ class TestUtils extends AbstractTestUtils { @override TestDefaultSqliteOpenFactory testFactory( {String? path, - String? sqlitePath, + String sqlitePath = defaultSqlitePath, SqliteOptions options = const SqliteOptions.defaults()}) { return TestSqliteOpenFactory( path: path ?? dbPath(), sqlitePath: sqlitePath, sqliteOptions: options); From 1243a5d1c65b18aef62d087e05ad50bdf66ff1f6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 10:41:21 +0200 Subject: [PATCH 33/57] split native and web tests. Add zone gaurds to web connections --- .../database/web_sqlite_connection_impl.dart | 26 +++- ...basic_test.dart => basic_native_test.dart} | 5 +- test/basic_shared_test.dart | 83 ++++++++++ test/utils/abstract_test_utils.dart | 8 +- test/utils/native_test_utils.dart | 7 +- test/utils/web_test_utils.dart | 38 +++-- test/watch_native_test.dart | 147 ++++++++++++++++++ ...watch_test.dart => watch_shared_test.dart} | 116 +++----------- 8 files changed, 303 insertions(+), 127 deletions(-) rename test/{basic_test.dart => basic_native_test.dart} (99%) create mode 100644 test/basic_shared_test.dart create mode 100644 test/watch_native_test.dart rename test/{watch_test.dart => watch_shared_test.dart} (69%) diff --git a/lib/src/web/database/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart index 0538944..2d5771c 100644 --- a/lib/src/web/database/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'package:meta/meta.dart'; +import 'package:sqlite_async/src/common/abstract_mutex.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; @@ -51,14 +52,35 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { Future readLock(Future Function(SqliteReadContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.lock(() => callback(WebReadContext(executor!))); + return _runZoned( + () => mutex.lock(() => callback(WebReadContext(executor!)), + timeout: lockTimeout), + debugContext: debugContext ?? 'execute()'); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, {Duration? lockTimeout, String? debugContext}) async { await isInitialized; - return mutex.lock(() => callback(WebWriteContext(executor!))); + return _runZoned( + () => mutex.lock(() => callback(WebWriteContext(executor!)), + timeout: lockTimeout), + debugContext: debugContext ?? 'execute()'); + } + + /// The [Mutex] on individual connections do already error in recursive locks. + /// + /// We duplicate the same check here, to: + /// 1. Also error when the recursive transaction is handled by a different + /// connection (with a different lock). + /// 2. Give a more specific error message when it happens. + T _runZoned(T Function() callback, {required String debugContext}) { + if (Zone.current[this] != null) { + throw LockError( + 'Recursive lock is not allowed. Use `tx.$debugContext` instead of `db.$debugContext`.'); + } + var zone = Zone.current.fork(zoneValues: {this: true}); + return zone.run(callback); } @override diff --git a/test/basic_test.dart b/test/basic_native_test.dart similarity index 99% rename from test/basic_test.dart rename to test/basic_native_test.dart index 6b0f279..6ba4573 100644 --- a/test/basic_test.dart +++ b/test/basic_native_test.dart @@ -1,3 +1,4 @@ +@TestOn('!browser') import 'dart:async'; import 'dart:math'; @@ -16,7 +17,6 @@ void main() { setUp(() async { path = testUtils.dbPath(); - await testUtils.init(); await testUtils.cleanDb(path: path); }); @@ -53,7 +53,8 @@ void main() { // Manually verified test('Concurrency', () async { - final db = SqliteDatabase.withFactory(testUtils.testFactory(path: path), + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path), maxReaders: 3); await db.initialize(); await createTables(db); diff --git a/test/basic_shared_test.dart b/test/basic_shared_test.dart new file mode 100644 index 0000000..c038ffb --- /dev/null +++ b/test/basic_shared_test.dart @@ -0,0 +1,83 @@ +import 'dart:async'; +import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:test/test.dart'; + +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + group('Shared Basic Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + tearDown(() async { + await testUtils.cleanDb(path: path); + }); + + createTables(SqliteDatabase db) async { + await db.writeTransaction((tx) async { + await tx.execute( + 'CREATE TABLE test_data(id INTEGER PRIMARY KEY AUTOINCREMENT, description TEXT)'); + }); + } + + test('should not allow direct db calls within a transaction callback', + () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + await db.writeTransaction((tx) async { + await expectLater(() async { + await db.execute( + 'INSERT INTO test_data(description) VALUES(?)', ['test']); + }, throwsA((e) => e is LockError && e.message.contains('tx.execute'))); + }); + }); + + test('should allow PRAMGAs', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + // Not allowed in transactions, but does work as a direct statement. + await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); + await db.execute('VACUUM'); + }); + + test('should allow ignoring errors', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + ignore(db.execute( + 'INSERT INTO test_data(description) VALUES(json(?))', ['test3'])); + }); + + test('should handle normal errors', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + Error? caughtError; + final syntheticError = ArgumentError('foobar'); + await db.writeLock((db) async { + throw syntheticError; + }).catchError((error) { + caughtError = error; + }); + expect(caughtError.toString(), equals(syntheticError.toString())); + + // Check that we can still continue afterwards + final computed = await db.writeLock((db) async { + return 5; + }); + expect(computed, equals(5)); + }); + }); +} + +// For some reason, future.ignore() doesn't actually ignore errors in these tests. +void ignore(Future future) { + future.then((_) {}, onError: (_) {}); +} diff --git a/test/utils/abstract_test_utils.dart b/test/utils/abstract_test_utils.dart index 609d000..6a6f094 100644 --- a/test/utils/abstract_test_utils.dart +++ b/test/utils/abstract_test_utils.dart @@ -19,23 +19,21 @@ abstract class AbstractTestUtils { } /// Generates a test open factory - TestDefaultSqliteOpenFactory testFactory( + Future testFactory( {String? path, String sqlitePath = '', - SqliteOptions options = const SqliteOptions.defaults()}) { + SqliteOptions options = const SqliteOptions.defaults()}) async { return TestDefaultSqliteOpenFactory( path: path ?? dbPath(), sqliteOptions: options); } /// Creates a SqliteDatabaseConnection Future setupDatabase({String? path}) async { - final db = SqliteDatabase.withFactory(testFactory(path: path)); + final db = SqliteDatabase.withFactory(await testFactory(path: path)); await db.initialize(); return db; } - Future init(); - /// Deletes any DB data Future cleanDb({required String path}); diff --git a/test/utils/native_test_utils.dart b/test/utils/native_test_utils.dart index bddafa0..d6a8615 100644 --- a/test/utils/native_test_utils.dart +++ b/test/utils/native_test_utils.dart @@ -49,9 +49,6 @@ class TestSqliteOpenFactory extends TestDefaultSqliteOpenFactory { } class TestUtils extends AbstractTestUtils { - @override - Future init() async {} - @override String dbPath() { Directory("test-db").createSync(recursive: false); @@ -88,10 +85,10 @@ class TestUtils extends AbstractTestUtils { } @override - TestDefaultSqliteOpenFactory testFactory( + Future testFactory( {String? path, String sqlitePath = defaultSqlitePath, - SqliteOptions options = const SqliteOptions.defaults()}) { + SqliteOptions options = const SqliteOptions.defaults()}) async { return TestSqliteOpenFactory( path: path ?? dbPath(), sqlitePath: sqlitePath, sqliteOptions: options); } diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index 380e8c4..101bea9 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -1,43 +1,51 @@ import 'dart:async'; +import 'dart:html'; +import 'package:js/js.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; import 'abstract_test_utils.dart'; +@JS('URL.createObjectURL') +external String _createObjectURL(Blob blob); + class TestUtils extends AbstractTestUtils { - late final Future _isInitialized; - late SqliteOptions? webOptions; + late Future _isInitialized; + late final SqliteOptions webOptions; TestUtils() { - _isInitialized = init(); + _isInitialized = _init(); } - @override - Future init() async { - if (webOptions != null) { - return; - } - + Future _init() async { final channel = spawnHybridUri('/test/server/asset_server.dart'); final port = await channel.stream.first as int; - final sqliteWasm = Uri.parse('http://localhost:$port/sqlite3.wasm'); - final sqliteDrift = Uri.parse('http://localhost:$port/drift_worker.js'); + final sqliteWasmUri = Uri.parse('http://localhost:$port/sqlite3.wasm'); + + // Cross origin workers are not supported, but we can supply a Blob + var sqliteDriftUri = + Uri.parse('http://localhost:$port/drift_worker.js').toString(); + final blob = Blob(['importScripts("$sqliteDriftUri");'], + 'application/javascript'); + sqliteDriftUri = _createObjectURL(blob); webOptions = SqliteOptions( webSqliteOptions: WebSqliteOptions( - wasmUri: sqliteWasm.toString(), workerUri: sqliteDrift.toString())); + wasmUri: sqliteWasmUri.toString(), + workerUri: sqliteDriftUri.toString())); } @override Future cleanDb({required String path}) async {} @override - TestDefaultSqliteOpenFactory testFactory( + Future testFactory( {String? path, String? sqlitePath, - SqliteOptions options = const SqliteOptions.defaults()}) { - return super.testFactory(path: path, options: webOptions!); + SqliteOptions options = const SqliteOptions.defaults()}) async { + await _isInitialized; + return super.testFactory(path: path, options: webOptions); } @override diff --git a/test/watch_native_test.dart b/test/watch_native_test.dart new file mode 100644 index 0000000..fc546fa --- /dev/null +++ b/test/watch_native_test.dart @@ -0,0 +1,147 @@ +@TestOn('!browser') +import 'dart:async'; +import 'dart:isolate'; +import 'dart:math'; + +import 'package:sqlite3/common.dart'; +import 'package:sqlite_async/sqlite_async.dart'; +import 'package:sqlite_async/src/utils/database_utils.dart'; +import 'package:test/test.dart'; + +import 'utils/test_utils_impl.dart'; + +final testUtils = TestUtils(); + +void main() { + createTables(AbstractSqliteDatabase db) async { + await db.writeTransaction((tx) async { + await tx.execute( + 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); + await tx.execute('CREATE INDEX assets_customer ON assets(customer_id)'); + await tx.execute( + 'CREATE TABLE customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute( + 'CREATE TABLE other_customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute('CREATE VIEW assets_alias AS SELECT * FROM assets'); + }); + } + + group('Query Watch Tests', () { + late String path; + + setUp(() async { + path = testUtils.dbPath(); + await testUtils.cleanDb(path: path); + }); + + for (var sqlite in testUtils.findSqliteLibraries()) { + test('getSourceTables - $sqlite', () async { + final db = SqliteDatabase.withFactory( + await testUtils.testFactory(path: path, sqlitePath: sqlite)); + await db.initialize(); + await createTables(db); + + var versionRow = await db.get('SELECT sqlite_version() as version'); + print('Testing SQLite ${versionRow['version']} - $sqlite'); + + final tables = await getSourceTables(db, + 'SELECT * FROM assets INNER JOIN customers ON assets.customer_id = customers.id'); + expect(tables, equals({'assets', 'customers'})); + + final tables2 = await getSourceTables(db, + 'SELECT count() FROM assets INNER JOIN "other_customers" AS oc ON assets.customer_id = oc.id AND assets.make = oc.name'); + expect(tables2, equals({'assets', 'other_customers'})); + + final tables3 = await getSourceTables(db, 'SELECT count() FROM assets'); + expect(tables3, equals({'assets'})); + + final tables4 = + await getSourceTables(db, 'SELECT count() FROM assets_alias'); + expect(tables4, equals({'assets'})); + + final tables5 = + await getSourceTables(db, 'SELECT sqlite_version() as version'); + expect(tables5, equals({})); + }); + } + + test('watch in isolate', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + const baseTime = 20; + + const throttleDuration = Duration(milliseconds: baseTime); + + final rows = await db.execute( + 'INSERT INTO customers(name) VALUES (?) RETURNING id', + ['a customer']); + final id = rows[0]['id']; + + var done = false; + inserts() async { + while (!done) { + await db.execute( + 'INSERT INTO assets(make, customer_id) VALUES (?, ?)', + ['test', id]); + await Future.delayed( + Duration(milliseconds: Random().nextInt(baseTime * 2))); + } + } + + const numberOfQueries = 10; + + inserts(); + + final factory = db.isolateConnectionFactory(); + + var l = await inIsolateWatch(factory, numberOfQueries, throttleDuration); + + var results = l[0] as List; + var times = l[1] as List; + done = true; + + var lastCount = 0; + for (var r in results) { + final count = r.first['count']; + // This is not strictly incrementing, since we can't guarantee the + // exact order between reads and writes. + // We can guarantee that there will always be a read after the last write, + // but the previous read may have been after the same write in some cases. + expect(count, greaterThanOrEqualTo(lastCount)); + lastCount = count; + } + + // The number of read queries must not be greater than the number of writes overall. + expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); + + DateTime? lastTime; + for (var r in times) { + if (lastTime != null) { + var diff = r.difference(lastTime); + expect(diff, greaterThanOrEqualTo(throttleDuration)); + } + lastTime = r; + } + }); + }); +} + +Future> inIsolateWatch(AbstractIsolateConnectionFactory factory, + int numberOfQueries, Duration throttleDuration) async { + return await Isolate.run(() async { + final db = factory.open(); + + final stream = db.watch( + 'SELECT count() AS count FROM assets INNER JOIN customers ON customers.id = assets.customer_id', + throttle: throttleDuration); + List times = []; + final results = await stream.take(numberOfQueries).map((e) { + times.add(DateTime.now()); + return e; + }).toList(); + + db.close(); + return [results, times]; + }); +} diff --git a/test/watch_test.dart b/test/watch_shared_test.dart similarity index 69% rename from test/watch_test.dart rename to test/watch_shared_test.dart index 5adbc83..a19d3b7 100644 --- a/test/watch_test.dart +++ b/test/watch_shared_test.dart @@ -1,42 +1,41 @@ import 'dart:async'; -import 'dart:isolate'; import 'dart:math'; - -import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/utils/database_utils.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:test/test.dart'; import 'utils/test_utils_impl.dart'; final testUtils = TestUtils(); -void main() { - createTables(AbstractSqliteDatabase db) async { - await db.writeTransaction((tx) async { - await tx.execute( - 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); - await tx.execute('CREATE INDEX assets_customer ON assets(customer_id)'); - await tx.execute( - 'CREATE TABLE customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); - await tx.execute( - 'CREATE TABLE other_customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); - await tx.execute('CREATE VIEW assets_alias AS SELECT * FROM assets'); - }); - } +createTables(AbstractSqliteDatabase db) async { + await db.writeTransaction((tx) async { + await tx.execute( + 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); + await tx.execute('CREATE INDEX assets_customer ON assets(customer_id)'); + await tx.execute( + 'CREATE TABLE customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute( + 'CREATE TABLE other_customers(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)'); + await tx.execute('CREATE VIEW assets_alias AS SELECT * FROM assets'); + }); +} +void main() { group('Query Watch Tests', () { late String path; + List sqlitePaths = []; setUp(() async { path = testUtils.dbPath(); + sqlitePaths = testUtils.findSqliteLibraries(); await testUtils.cleanDb(path: path); }); - for (var sqlite in testUtils.findSqliteLibraries()) { + for (var sqlite in sqlitePaths) { test('getSourceTables - $sqlite', () async { final db = SqliteDatabase.withFactory( - testUtils.testFactory(path: path, sqlitePath: sqlite)); + await testUtils.testFactory(path: path, sqlitePath: sqlite)); await db.initialize(); await createTables(db); @@ -188,66 +187,6 @@ void main() { expect(events, equals([UpdateNotification.single('assets')])); }); - test('watch in isolate', () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); - - const baseTime = 20; - - const throttleDuration = Duration(milliseconds: baseTime); - - final rows = await db.execute( - 'INSERT INTO customers(name) VALUES (?) RETURNING id', - ['a customer']); - final id = rows[0]['id']; - - var done = false; - inserts() async { - while (!done) { - await db.execute( - 'INSERT INTO assets(make, customer_id) VALUES (?, ?)', - ['test', id]); - await Future.delayed( - Duration(milliseconds: Random().nextInt(baseTime * 2))); - } - } - - const numberOfQueries = 10; - - inserts(); - - final factory = db.isolateConnectionFactory(); - - var l = await inIsolateWatch(factory, numberOfQueries, throttleDuration); - - var results = l[0] as List; - var times = l[1] as List; - done = true; - - var lastCount = 0; - for (var r in results) { - final count = r.first['count']; - // This is not strictly incrementing, since we can't guarantee the - // exact order between reads and writes. - // We can guarantee that there will always be a read after the last write, - // but the previous read may have been after the same write in some cases. - expect(count, greaterThanOrEqualTo(lastCount)); - lastCount = count; - } - - // The number of read queries must not be greater than the number of writes overall. - expect(numberOfQueries, lessThanOrEqualTo(results.last.first['count'])); - - DateTime? lastTime; - for (var r in times) { - if (lastTime != null) { - var diff = r.difference(lastTime); - expect(diff, greaterThanOrEqualTo(throttleDuration)); - } - lastTime = r; - } - }); - test('watch with parameters', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); @@ -315,22 +254,3 @@ void main() { }); }); } - -Future> inIsolateWatch(AbstractIsolateConnectionFactory factory, - int numberOfQueries, Duration throttleDuration) async { - return await Isolate.run(() async { - final db = factory.open(); - - final stream = db.watch( - 'SELECT count() AS count FROM assets INNER JOIN customers ON customers.id = assets.customer_id', - throttle: throttleDuration); - List times = []; - final results = await stream.take(numberOfQueries).map((e) { - times.add(DateTime.now()); - return e; - }).toList(); - - db.close(); - return [results, times]; - }); -} From 9f16eff71dd233516bdf24f487ac91734d88ed1a Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 11:51:49 +0200 Subject: [PATCH 34/57] run web tests in CI --- .github/workflows/test.yaml | 7 +++++-- lib/drift.dart | 1 + lib/src/web/worker/drift_worker.dart | 19 +++++++++++++++++++ lib/src/web/worker/worker_utils.dart | 0 test/json1_test.dart | 12 ++++++++---- test/server/worker_server.dart | 7 +++---- test/utils/stub_test_utils.dart | 5 ----- test/utils/web_test_utils.dart | 2 +- test/watch_shared_test.dart | 2 ++ 9 files changed, 39 insertions(+), 16 deletions(-) create mode 100644 lib/src/web/worker/drift_worker.dart create mode 100644 lib/src/web/worker/worker_utils.dart diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9fa2df..de91e97 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,9 +55,12 @@ jobs: run: dart pub get - name: Install SQLite - run: ./scripts/install_sqlite.sh ${{ matrix.sqlite_version }} ${{ matrix.sqlite_url }} + run: | + ./scripts/install_sqlite.sh ${{ matrix.sqlite_version }} ${{ matrix.sqlite_url }} + mkdir -p assets && curl -LJ https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.3.0/sqlite3.wasm -o assets/sqlite3.wasm + - name: Run Tests run: | export LD_LIBRARY_PATH=./sqlite-autoconf-${{ matrix.sqlite_version }}/.libs - dart test + dart test -p vm,chrome diff --git a/lib/drift.dart b/lib/drift.dart index e4cb41f..9e038a3 100644 --- a/lib/drift.dart +++ b/lib/drift.dart @@ -3,3 +3,4 @@ library; export 'package:drift/wasm.dart'; +export 'package:sqlite_async/src/web/worker/worker_utils.dart'; diff --git a/lib/src/web/worker/drift_worker.dart b/lib/src/web/worker/drift_worker.dart new file mode 100644 index 0000000..4f28b16 --- /dev/null +++ b/lib/src/web/worker/drift_worker.dart @@ -0,0 +1,19 @@ +/// This is an example of a database worker script +/// Custom database logic can be achieved by implementing this template +/// This file needs to be compiled to JavaScript with the command: +/// dart compile js -O4 lib/src/web/worker/drift_worker.dart -o build/drift_worker.js +/// The output should then be included in each project's `web` directory +library; + +import 'package:sqlite_async/drift.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; + +/// Use this function to register any custom DB functionality +/// which requires direct access to the connection +void setupDatabase(CommonDatabase database) {} + +void main() { + WasmDatabase.workerMainForOpen( + setupAllDatabases: setupDatabase, + ); +} diff --git a/lib/src/web/worker/worker_utils.dart b/lib/src/web/worker/worker_utils.dart new file mode 100644 index 0000000..e69de29 diff --git a/test/json1_test.dart b/test/json1_test.dart index 90b91d6..8e0ed38 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; @@ -44,7 +46,6 @@ void main() { test('Inserts', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); - var users1 = [ TestUser(name: 'Bob', email: 'bob@example.org'), TestUser(name: 'Alice', email: 'alice@example.org') @@ -53,27 +54,30 @@ void main() { TestUser(name: 'Charlie', email: 'charlie@example.org'), TestUser(name: 'Dan', email: 'dan@example.org') ]; + + print(jsonEncode(users1)); var ids1 = await db.execute( "INSERT INTO users(name, email) SELECT e.value ->> 'name', e.value ->> 'email' FROM json_each(?) e RETURNING id", - [users1]); + [jsonEncode(users1)]); var ids2 = await db.execute( "INSERT INTO users(name, email) ${selectJsonColumns([ 'name', 'email' ])} RETURNING id", - [users2]); + [jsonEncode(users2)]); var ids = [ for (var row in ids1) row, for (var row in ids2) row, ]; + var results = [ for (var row in await db.getAll( "SELECT id, name, email FROM users WHERE id IN (${selectJsonColumns([ 'id' ])}) ORDER BY name", - [ids])) + [jsonEncode(ids)])) TestUser.fromMap(row) ]; diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart index f581005..efa56aa 100644 --- a/test/server/worker_server.dart +++ b/test/server/worker_server.dart @@ -15,17 +15,16 @@ Future hybridMain(StreamChannel channel) async { .resolveSymbolicLinksSync(); // Copy sqlite3.wasm file expected by the worker - await File('example/web/sqlite3.wasm') - .copy(p.join(directory, 'sqlite3.wasm')); + await File('assets/sqlite3.wasm').copy(p.join(directory, 'sqlite3.wasm')); // And compile worker code final process = await Process.run(Platform.executable, [ 'compile', 'js', '-o', - p.join(directory, 'worker.dart.js'), + p.join(directory, 'drift_worker.js'), '-O0', - 'test/wasm/worker.dart', + 'lib/src/web/worker/drift_worker.dart', ]); if (process.exitCode != 0) { diff --git a/test/utils/stub_test_utils.dart b/test/utils/stub_test_utils.dart index a9ccf55..5e3a953 100644 --- a/test/utils/stub_test_utils.dart +++ b/test/utils/stub_test_utils.dart @@ -10,9 +10,4 @@ class TestUtils extends AbstractTestUtils { List findSqliteLibraries() { throw UnimplementedError(); } - - @override - Future init() { - throw UnimplementedError(); - } } diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index 101bea9..cfc4c3d 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -18,7 +18,7 @@ class TestUtils extends AbstractTestUtils { } Future _init() async { - final channel = spawnHybridUri('/test/server/asset_server.dart'); + final channel = spawnHybridUri('/test/server/worker_server.dart'); final port = await channel.stream.first as int; final sqliteWasmUri = Uri.parse('http://localhost:$port/sqlite3.wasm'); diff --git a/test/watch_shared_test.dart b/test/watch_shared_test.dart index a19d3b7..9155766 100644 --- a/test/watch_shared_test.dart +++ b/test/watch_shared_test.dart @@ -1,3 +1,5 @@ +@TestOn('!browser') +// TODO watched queries require forked Drift lib worker import 'dart:async'; import 'dart:math'; import 'package:sqlite_async/sqlite_async.dart'; From 23e211a09c20a3415ff68892a90f1af458318a66 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 12:07:15 +0200 Subject: [PATCH 35/57] test: only compile assets once --- lib/src/web/worker/worker_utils.dart | 3 +++ test/server/worker_server.dart | 33 ++++++++++++++++------------ test/utils/web_test_utils.dart | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/lib/src/web/worker/worker_utils.dart b/lib/src/web/worker/worker_utils.dart index e69de29..4bb335c 100644 --- a/lib/src/web/worker/worker_utils.dart +++ b/lib/src/web/worker/worker_utils.dart @@ -0,0 +1,3 @@ +import 'package:sqlite_async/sqlite3_common.dart'; + +void setupCommonWorkerDB(CommonDatabase database) {} diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart index efa56aa..3f3c411 100644 --- a/test/server/worker_server.dart +++ b/test/server/worker_server.dart @@ -15,20 +15,25 @@ Future hybridMain(StreamChannel channel) async { .resolveSymbolicLinksSync(); // Copy sqlite3.wasm file expected by the worker - await File('assets/sqlite3.wasm').copy(p.join(directory, 'sqlite3.wasm')); - - // And compile worker code - final process = await Process.run(Platform.executable, [ - 'compile', - 'js', - '-o', - p.join(directory, 'drift_worker.js'), - '-O0', - 'lib/src/web/worker/drift_worker.dart', - ]); - - if (process.exitCode != 0) { - fail('Could not compile worker'); + final sqliteOutputPath = p.join(directory, 'sqlite3.wasm'); + if (!(await File(sqliteOutputPath).exists())) { + await File('assets/sqlite3.wasm').copy(sqliteOutputPath); + } + + final driftWorkerPath = p.join(directory, 'drift_worker.js'); + if (!(await File(driftWorkerPath).exists())) { + // And compile worker code + final process = await Process.run(Platform.executable, [ + 'compile', + 'js', + '-o', + driftWorkerPath, + '-O0', + 'lib/src/web/worker/drift_worker.dart', + ]); + if (process.exitCode != 0) { + fail('Could not compile worker'); + } } final server = await HttpServer.bind('localhost', 0); diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index cfc4c3d..98eed43 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -20,7 +20,7 @@ class TestUtils extends AbstractTestUtils { Future _init() async { final channel = spawnHybridUri('/test/server/worker_server.dart'); final port = await channel.stream.first as int; - + print('configure tests'); final sqliteWasmUri = Uri.parse('http://localhost:$port/sqlite3.wasm'); // Cross origin workers are not supported, but we can supply a Blob From e591cdcab3fb490841aaf5972273457dda7d7d89 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 12:46:51 +0200 Subject: [PATCH 36/57] compile worker only once --- .github/workflows/test.yaml | 2 ++ test/server/worker_server.dart | 31 +++++++++++++++---------------- test/utils/web_test_utils.dart | 14 ++++++-------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index de91e97..e1431ff 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,6 +59,8 @@ jobs: ./scripts/install_sqlite.sh ${{ matrix.sqlite_version }} ${{ matrix.sqlite_url }} mkdir -p assets && curl -LJ https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.3.0/sqlite3.wasm -o assets/sqlite3.wasm + - name: Compile WebWorker + run: dart compile -o assets/drift_worker.js -O0 lib/src/web/worker/drift_worker.dart - name: Run Tests run: | diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart index 3f3c411..ad8d1ac 100644 --- a/test/server/worker_server.dart +++ b/test/server/worker_server.dart @@ -16,26 +16,24 @@ Future hybridMain(StreamChannel channel) async { // Copy sqlite3.wasm file expected by the worker final sqliteOutputPath = p.join(directory, 'sqlite3.wasm'); - if (!(await File(sqliteOutputPath).exists())) { - await File('assets/sqlite3.wasm').copy(sqliteOutputPath); - } + await File('assets/sqlite3.wasm').copy(sqliteOutputPath); final driftWorkerPath = p.join(directory, 'drift_worker.js'); - if (!(await File(driftWorkerPath).exists())) { - // And compile worker code - final process = await Process.run(Platform.executable, [ - 'compile', - 'js', - '-o', - driftWorkerPath, - '-O0', - 'lib/src/web/worker/drift_worker.dart', - ]); - if (process.exitCode != 0) { - fail('Could not compile worker'); - } + // And compile worker code + final process = await Process.run(Platform.executable, [ + 'compile', + 'js', + '-o', + driftWorkerPath, + '-O0', + 'lib/src/web/worker/drift_worker.dart', + ]); + if (process.exitCode != 0) { + fail('Could not compile worker'); } + print('compiled worker'); + final server = await HttpServer.bind('localhost', 0); final handler = const Pipeline() @@ -45,6 +43,7 @@ Future hybridMain(StreamChannel channel) async { channel.sink.add(server.port); await channel.stream.listen(null).asFuture().then((_) async { + print('closing server'); await server.close(); await Directory(directory).delete(); }); diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index 98eed43..1def507 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -18,22 +18,20 @@ class TestUtils extends AbstractTestUtils { } Future _init() async { - final channel = spawnHybridUri('/test/server/worker_server.dart'); + final channel = spawnHybridUri('/test/server/asset_server.dart'); final port = await channel.stream.first as int; - print('configure tests'); - final sqliteWasmUri = Uri.parse('http://localhost:$port/sqlite3.wasm'); - + final sqliteWasmUri = 'http://localhost:$port/sqlite3.wasm'; // Cross origin workers are not supported, but we can supply a Blob - var sqliteDriftUri = - Uri.parse('http://localhost:$port/drift_worker.js').toString(); + var sqliteDriftUri = 'http://localhost:$port/drift_worker.js'; + + print('Using $sqliteDriftUri and $sqliteDriftUri'); final blob = Blob(['importScripts("$sqliteDriftUri");'], 'application/javascript'); sqliteDriftUri = _createObjectURL(blob); webOptions = SqliteOptions( webSqliteOptions: WebSqliteOptions( - wasmUri: sqliteWasmUri.toString(), - workerUri: sqliteDriftUri.toString())); + wasmUri: sqliteWasmUri.toString(), workerUri: sqliteDriftUri)); } @override From c001b3bc295744214aba98f60aff66a11bd03a1c Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 12:49:56 +0200 Subject: [PATCH 37/57] fix compile cmd typo --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e1431ff..a41c669 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -60,7 +60,7 @@ jobs: mkdir -p assets && curl -LJ https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.3.0/sqlite3.wasm -o assets/sqlite3.wasm - name: Compile WebWorker - run: dart compile -o assets/drift_worker.js -O0 lib/src/web/worker/drift_worker.dart + run: dart compile js -o assets/drift_worker.js -O0 lib/src/web/worker/drift_worker.dart - name: Run Tests run: | From 6ffb12993db3b24e729f6276140c5b0e6834d04c Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 14:06:20 +0200 Subject: [PATCH 38/57] cleanup tests --- test/server/worker_server.dart | 43 +++++++++++++++++----------------- test/utils/web_test_utils.dart | 3 +-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart index ad8d1ac..53a3011 100644 --- a/test/server/worker_server.dart +++ b/test/server/worker_server.dart @@ -10,41 +10,42 @@ import 'package:test/test.dart'; import 'asset_server.dart'; Future hybridMain(StreamChannel channel) async { - final directory = Directory.systemTemp - .createTempSync('sqlite_dart_web') - .resolveSymbolicLinksSync(); + final directory = Directory('./assets'); // Copy sqlite3.wasm file expected by the worker - final sqliteOutputPath = p.join(directory, 'sqlite3.wasm'); - await File('assets/sqlite3.wasm').copy(sqliteOutputPath); - - final driftWorkerPath = p.join(directory, 'drift_worker.js'); - // And compile worker code - final process = await Process.run(Platform.executable, [ - 'compile', - 'js', - '-o', - driftWorkerPath, - '-O0', - 'lib/src/web/worker/drift_worker.dart', - ]); - if (process.exitCode != 0) { - fail('Could not compile worker'); + final sqliteOutputPath = p.join(directory.path, 'sqlite3.wasm'); + + if (!(await File(sqliteOutputPath).exists())) { + throw AssertionError( + 'sqlite3.wasm file should be present in the ./assets folder'); } - print('compiled worker'); + final driftWorkerPath = p.join(directory.path, 'drift_worker.js'); + if (!(await File(driftWorkerPath).exists())) { + // And compile worker code + final process = await Process.run(Platform.executable, [ + 'compile', + 'js', + '-o', + driftWorkerPath, + '-O0', + 'lib/src/web/worker/drift_worker.dart', + ]); + if (process.exitCode != 0) { + fail('Could not compile worker'); + } + } final server = await HttpServer.bind('localhost', 0); final handler = const Pipeline() .addMiddleware(cors()) - .addHandler(createStaticHandler(directory)); + .addHandler(createStaticHandler(directory.path)); io.serveRequests(server, handler); channel.sink.add(server.port); await channel.stream.listen(null).asFuture().then((_) async { print('closing server'); await server.close(); - await Directory(directory).delete(); }); } diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index 1def507..9ba47ec 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -18,13 +18,12 @@ class TestUtils extends AbstractTestUtils { } Future _init() async { - final channel = spawnHybridUri('/test/server/asset_server.dart'); + final channel = spawnHybridUri('/test/server/worker_server.dart'); final port = await channel.stream.first as int; final sqliteWasmUri = 'http://localhost:$port/sqlite3.wasm'; // Cross origin workers are not supported, but we can supply a Blob var sqliteDriftUri = 'http://localhost:$port/drift_worker.js'; - print('Using $sqliteDriftUri and $sqliteDriftUri'); final blob = Blob(['importScripts("$sqliteDriftUri");'], 'application/javascript'); sqliteDriftUri = _createObjectURL(blob); From 02c2d9fa8c6a4a2f01cd70a1f0802f88c32b3ec0 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 1 Feb 2024 15:43:17 +0200 Subject: [PATCH 39/57] improve web locks and transactions according to test spec --- lib/src/web/database/web_db_context.dart | 50 ++++++++-- .../database/web_sqlite_connection_impl.dart | 64 ++++++++++-- lib/src/web/database/web_sqlite_database.dart | 16 +++ lib/src/web/worker/drift_worker.dart | 4 +- lib/src/web/worker/worker_utils.dart | 11 ++- test/basic_native_test.dart | 99 ------------------- test/basic_shared_test.dart | 73 ++++++++++++++ test/json1_test.dart | 1 - 8 files changed, 201 insertions(+), 117 deletions(-) diff --git a/lib/src/web/database/web_db_context.dart b/lib/src/web/database/web_db_context.dart index b2d563e..bd89c90 100644 --- a/lib/src/web/database/web_db_context.dart +++ b/lib/src/web/database/web_db_context.dart @@ -1,13 +1,21 @@ import 'dart:async'; +import 'package:meta/meta.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'executor/sqlite_executor.dart'; +/// Custom function which exposes CommonDatabase.autocommit +const sqliteAsyncAutoCommitCommand = 'sqlite_async_autocommit'; + class WebReadContext implements SqliteReadContext { SQLExecutor db; + bool _closed = false; + + @protected + bool isTransaction; - WebReadContext(this.db); + WebReadContext(this.db, {this.isTransaction = false}); @override Future computeWithDatabase( @@ -17,12 +25,15 @@ class WebReadContext implements SqliteReadContext { @override Future get(String sql, [List parameters = const []]) async { - return (await db.select(sql, parameters)).first; + return (await getAll(sql, parameters)).first; } @override Future getAll(String sql, [List parameters = const []]) async { + if (_closed) { + throw SqliteException(0, 'Transaction closed', null, sql); + } return db.select(sql, parameters); } @@ -30,27 +41,52 @@ class WebReadContext implements SqliteReadContext { Future getOptional(String sql, [List parameters = const []]) async { try { - return (await db.select(sql, parameters)).first; + return (await getAll(sql, parameters)).first; } catch (ex) { return null; } } @override - bool get closed => throw UnimplementedError(); + bool get closed => _closed; + + close() { + _closed = true; + } @override - Future getAutoCommit() { - throw UnimplementedError(); + Future getAutoCommit() async { + final response = await db.select('select $sqliteAsyncAutoCommitCommand()'); + if (response.isEmpty) { + return false; + } + + return response.first.values.first != 0; } } class WebWriteContext extends WebReadContext implements SqliteWriteContext { - WebWriteContext(super.db); + WebWriteContext(super.db, {super.isTransaction}); @override Future execute(String sql, [List parameters = const []]) async { + return getAll(sql, parameters); + } + + @override + Future getAll(String sql, + [List parameters = const []]) async { + if (_closed) { + throw SqliteException(0, 'Transaction closed', null, sql); + } + final isAutoCommit = await getAutoCommit(); + + /// Statements in read/writeTransactions should not execute after ROLLBACK + if (isTransaction && !sql.toLowerCase().contains('begin') && isAutoCommit) { + throw SqliteException(0, + 'Transaction rolled back by earlier statement. Cannot execute: $sql'); + } return db.select(sql, parameters); } diff --git a/lib/src/web/database/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart index 2d5771c..041257b 100644 --- a/lib/src/web/database/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -6,6 +6,7 @@ import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; import 'package:sqlite_async/src/web/web_mutex.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; @@ -50,24 +51,70 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { @override Future readLock(Future Function(SqliteReadContext tx) callback, - {Duration? lockTimeout, String? debugContext}) async { + {Duration? lockTimeout, + String? debugContext, + bool isTransaction = false}) async { await isInitialized; return _runZoned( - () => mutex.lock(() => callback(WebReadContext(executor!)), - timeout: lockTimeout), + () => mutex.lock(() async { + final context = + WebReadContext(executor!, isTransaction: isTransaction); + try { + final result = await callback(context); + return result; + } finally { + context.close(); + } + }, timeout: lockTimeout), debugContext: debugContext ?? 'execute()'); } @override Future writeLock(Future Function(SqliteWriteContext tx) callback, - {Duration? lockTimeout, String? debugContext}) async { + {Duration? lockTimeout, + String? debugContext, + bool isTransaction = false}) async { await isInitialized; return _runZoned( - () => mutex.lock(() => callback(WebWriteContext(executor!)), - timeout: lockTimeout), + () => mutex.lock(() async { + final context = + WebWriteContext(executor!, isTransaction: isTransaction); + try { + final result = await callback(context); + return result; + } finally { + context.close(); + } + }, timeout: lockTimeout), debugContext: debugContext ?? 'execute()'); } + @override + Future readTransaction( + Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout}) async { + return readLock((ctx) async { + return await internalReadTransaction(ctx, callback); + }, + lockTimeout: lockTimeout, + debugContext: 'readTransaction()', + isTransaction: true); + } + + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout}) async { + return writeLock(( + ctx, + ) async { + return await internalWriteTransaction(ctx, callback); + }, + lockTimeout: lockTimeout, + debugContext: 'writeTransaction()', + isTransaction: true); + } + /// The [Mutex] on individual connections do already error in recursive locks. /// /// We duplicate the same check here, to: @@ -90,7 +137,8 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { } @override - Future getAutoCommit() { - throw UnimplementedError(); + Future getAutoCommit() async { + await isInitialized; + return WebWriteContext(executor!).getAutoCommit(); } } diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index 99f33a0..b6aee81 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -90,6 +90,22 @@ class SqliteDatabase extends AbstractSqliteDatabase { lockTimeout: lockTimeout, debugContext: debugContext); } + @override + Future readTransaction( + Future Function(SqliteReadContext tx) callback, + {Duration? lockTimeout, + String? debugContext}) async { + return _connection.readTransaction(callback, lockTimeout: lockTimeout); + } + + @override + Future writeTransaction( + Future Function(SqliteWriteContext tx) callback, + {Duration? lockTimeout, + String? debugContext}) async { + return _connection.writeTransaction(callback, lockTimeout: lockTimeout); + } + @override Future close() async { return _connection.close(); diff --git a/lib/src/web/worker/drift_worker.dart b/lib/src/web/worker/drift_worker.dart index 4f28b16..51860b7 100644 --- a/lib/src/web/worker/drift_worker.dart +++ b/lib/src/web/worker/drift_worker.dart @@ -10,7 +10,9 @@ import 'package:sqlite_async/sqlite3_common.dart'; /// Use this function to register any custom DB functionality /// which requires direct access to the connection -void setupDatabase(CommonDatabase database) {} +void setupDatabase(CommonDatabase database) { + setupCommonWorkerDB(database); +} void main() { WasmDatabase.workerMainForOpen( diff --git a/lib/src/web/worker/worker_utils.dart b/lib/src/web/worker/worker_utils.dart index 4bb335c..cf5b611 100644 --- a/lib/src/web/worker/worker_utils.dart +++ b/lib/src/web/worker/worker_utils.dart @@ -1,3 +1,12 @@ import 'package:sqlite_async/sqlite3_common.dart'; +import 'package:sqlite_async/src/web/database/web_db_context.dart'; -void setupCommonWorkerDB(CommonDatabase database) {} +void setupCommonWorkerDB(CommonDatabase database) { + /// Exposes autocommit via a query function + database.createFunction( + functionName: sqliteAsyncAutoCommitCommand, + argumentCount: const AllowedArgumentCount(0), + function: (args) { + return database.autocommit; + }); +} diff --git a/test/basic_native_test.dart b/test/basic_native_test.dart index 6ba4573..840fe1e 100644 --- a/test/basic_native_test.dart +++ b/test/basic_native_test.dart @@ -205,14 +205,6 @@ void main() { // }); }); - test('should allow PRAMGAs', () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); - // Not allowed in transactions, but does work as a direct statement. - await db.execute('PRAGMA wal_checkpoint(TRUNCATE)'); - await db.execute('VACUUM'); - }); - test('should allow ignoring errors', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); @@ -221,50 +213,6 @@ void main() { 'INSERT INTO test_data(description) VALUES(json(?))', ['test3'])); }); - test('should properly report errors in transactions', () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); - - var tp = db.writeTransaction((tx) async { - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [1, 'test1']); - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [2, 'test2']); - expect(await tx.getAutoCommit(), equals(false)); - try { - await tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [2, 'test3']); - } catch (e) { - // Ignore - } - expect(await tx.getAutoCommit(), equals(true)); - expect(tx.closed, equals(false)); - - // Will not be executed because of the above rollback - ignore(tx.execute( - 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', - [4, 'test4'])); - }); - - // The error propagates up to the transaction - await expectLater( - tp, - throwsA((e) => - e is sqlite.SqliteException && - e.message - .contains('Transaction rolled back by earlier statement') && - e.message.contains('UNIQUE constraint failed'))); - - expect(await db.get('SELECT count() count FROM test_data'), - equals({'count': 0})); - - // Check that we can open another transaction afterwards - await db.writeTransaction((tx) async {}); - }); - test('should error on dangling transactions', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); @@ -273,25 +221,6 @@ void main() { }, throwsA((e) => e is sqlite.SqliteException)); }); - test('should handle normal errors', () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); - Error? caughtError; - final syntheticError = ArgumentError('foobar'); - await db.computeWithDatabase((db) async { - throw syntheticError; - }).catchError((error) { - caughtError = error; - }); - expect(caughtError.toString(), equals(syntheticError.toString())); - - // Check that we can still continue afterwards - final computed = await db.computeWithDatabase((db) async { - return 5; - }); - expect(computed, equals(5)); - }); - test('should handle uncaught errors', () async { final db = await testUtils.setupDatabase(path: path); await createTables(db); @@ -344,34 +273,6 @@ void main() { }); expect(computed, equals(5)); }); - - test('should allow resuming transaction after errors', () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); - SqliteWriteContext? savedTx; - await db.writeTransaction((tx) async { - savedTx = tx; - var caught = false; - try { - // This error does not rollback the transaction - await tx.execute('NOT A VALID STATEMENT'); - } catch (e) { - // Ignore - caught = true; - } - expect(caught, equals(true)); - - expect(await tx.getAutoCommit(), equals(false)); - expect(tx.closed, equals(false)); - - final rs = await tx.execute( - 'INSERT INTO test_data(description) VALUES(?) RETURNING description', - ['Test Data']); - expect(rs.rows[0], equals(['Test Data'])); - }); - expect(await savedTx!.getAutoCommit(), equals(true)); - expect(savedTx!.closed, equals(true)); - }); }); } diff --git a/test/basic_shared_test.dart b/test/basic_shared_test.dart index c038ffb..bd3e128 100644 --- a/test/basic_shared_test.dart +++ b/test/basic_shared_test.dart @@ -1,5 +1,6 @@ import 'dart:async'; import 'package:sqlite_async/mutex.dart'; +import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; @@ -74,6 +75,78 @@ void main() { }); expect(computed, equals(5)); }); + + test('should allow resuming transaction after errors', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + SqliteWriteContext? savedTx; + await db.writeTransaction((tx) async { + savedTx = tx; + var caught = false; + try { + // This error does not rollback the transaction + await tx.execute('NOT A VALID STATEMENT'); + } catch (e) { + // Ignore + caught = true; + } + expect(caught, equals(true)); + + expect(await tx.getAutoCommit(), equals(false)); + expect(tx.closed, equals(false)); + + final rs = await tx.execute( + 'INSERT INTO test_data(description) VALUES(?) RETURNING description', + ['Test Data']); + expect(rs.rows[0], equals(['Test Data'])); + }); + expect(await savedTx!.getAutoCommit(), equals(true)); + expect(savedTx!.closed, equals(true)); + }); + + test('should properly report errors in transactions', () async { + final db = await testUtils.setupDatabase(path: path); + await createTables(db); + + var tp = db.writeTransaction((tx) async { + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [1, 'test1']); + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [2, 'test2']); + expect(await tx.getAutoCommit(), equals(false)); + try { + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [2, 'test3']); + } catch (e) { + // Ignore + } + + expect(await tx.getAutoCommit(), equals(true)); + expect(tx.closed, equals(false)); + + // Will not be executed because of the above rollback + await tx.execute( + 'INSERT OR ROLLBACK INTO test_data(id, description) VALUES(?, ?)', + [4, 'test4']); + }); + + // The error propagates up to the transaction + await expectLater( + tp, + throwsA((e) => + e is SqliteException && + e.message + .contains('Transaction rolled back by earlier statement'))); + + expect(await db.get('SELECT count() count FROM test_data'), + equals({'count': 0})); + + // Check that we can open another transaction afterwards + await db.writeTransaction((tx) async {}); + }); }); } diff --git a/test/json1_test.dart b/test/json1_test.dart index 8e0ed38..7717d43 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -55,7 +55,6 @@ void main() { TestUser(name: 'Dan', email: 'dan@example.org') ]; - print(jsonEncode(users1)); var ids1 = await db.execute( "INSERT INTO users(name, email) SELECT e.value ->> 'name', e.value ->> 'email' FROM json_each(?) e RETURNING id", [jsonEncode(users1)]); From 1145db3b9a06f580c0867c6960cd67eefff7ec6a Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Fri, 2 Feb 2024 08:55:15 +0200 Subject: [PATCH 40/57] Convert Drift SQL exceptions to SQLite exceptions --- .../connection/sync_sqlite_connection.dart | 5 ++- .../database/executor/drift_sql_executor.dart | 40 ++++++++++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart index db9eb8b..648fa9a 100644 --- a/lib/src/common/connection/sync_sqlite_connection.dart +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -8,13 +8,14 @@ import 'package:sqlite_async/src/update_notification.dart'; /// implementation using a synchronous connection class SyncSqliteConnection extends SqliteConnection with SqliteQueries { final CommonDatabase db; - AbstractMutex mutex; + late AbstractMutex mutex; @override late final Stream updates; bool _closed = false; - SyncSqliteConnection(this.db, this.mutex) { + SyncSqliteConnection(this.db, AbstractMutex m) { + mutex = m.open(); updates = db.updates.map( (event) { return UpdateNotification({event.tableName}); diff --git a/lib/src/web/database/executor/drift_sql_executor.dart b/lib/src/web/database/executor/drift_sql_executor.dart index 9ff5f4c..8911e41 100644 --- a/lib/src/web/database/executor/drift_sql_executor.dart +++ b/lib/src/web/database/executor/drift_sql_executor.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:drift/drift.dart'; +import 'package:drift/remote.dart'; import 'package:drift/wasm.dart'; import 'package:sqlite3/common.dart'; import 'sqlite_executor.dart'; @@ -27,20 +28,41 @@ class DriftWebSQLExecutor extends SQLExecutor { } @override - Future executeBatch(String sql, List> parameterSets) { - return db.resolvedExecutor.runBatched(BatchedStatements([sql], - parameterSets.map((e) => ArgumentsForBatchedStatement(0, e)).toList())); + Future executeBatch( + String sql, List> parameterSets) async { + try { + final result = await db.resolvedExecutor.runBatched(BatchedStatements( + [sql], + parameterSets + .map((e) => ArgumentsForBatchedStatement(0, e)) + .toList())); + return result; + } on DriftRemoteException catch (e) { + if (e.toString().contains('SqliteException')) { + // Drift wraps these in remote errors + throw SqliteException(e.remoteCause.hashCode, e.remoteCause.toString()); + } + rethrow; + } } @override - FutureOr select(String sql, + Future select(String sql, [List parameters = const []]) async { - final result = await db.resolvedExecutor.runSelect(sql, parameters); - if (result.isEmpty) { - return ResultSet([], [], []); + try { + final result = await db.resolvedExecutor.runSelect(sql, parameters); + if (result.isEmpty) { + return ResultSet([], [], []); + } + return ResultSet(result.first.keys.toList(), [], + result.map((e) => e.values.toList()).toList()); + } on DriftRemoteException catch (e) { + if (e.toString().contains('SqliteException')) { + // Drift wraps these in remote errors + throw SqliteException(e.remoteCause.hashCode, e.remoteCause.toString()); + } + rethrow; } - return ResultSet(result.first.keys.toList(), [], - result.map((e) => e.values.toList()).toList()); } } From 70c2324c497f9af49d5c310e6c76d6f0510bf5dc Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 5 Feb 2024 15:09:57 +0200 Subject: [PATCH 41/57] Allow autocommit on web database connection --- lib/src/web/database/web_sqlite_database.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index b6aee81..3cc5596 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -118,6 +118,6 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override Future getAutoCommit() { - throw UnimplementedError(); + return _connection.getAutoCommit(); } } From 3e4bdd4b43afb6b5875464d35f61b2ed8bd9c3f6 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Mon, 5 Feb 2024 15:13:11 +0200 Subject: [PATCH 42/57] remove duplicate test --- test/basic_native_test.dart | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/basic_native_test.dart b/test/basic_native_test.dart index 840fe1e..eee324c 100644 --- a/test/basic_native_test.dart +++ b/test/basic_native_test.dart @@ -106,19 +106,6 @@ void main() { }); }); - test('should not allow direct db calls within a transaction callback', - () async { - final db = await testUtils.setupDatabase(path: path); - await createTables(db); - - await db.writeTransaction((tx) async { - await expectLater(() async { - await db.execute( - 'INSERT INTO test_data(description) VALUES(?)', ['test']); - }, throwsA((e) => e is LockError && e.message.contains('tx.execute'))); - }); - }); - test('should not allow read-only db calls within transaction callback', () async { final db = await testUtils.setupDatabase(path: path); From 5307003e1f6134288be4d81744bb69bd860fe731 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 6 Feb 2024 13:04:48 +0200 Subject: [PATCH 43/57] use forked version of Drift --- pubspec.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index c065f8d..a3fd7d2 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,7 +6,12 @@ environment: sdk: '>=3.2.0 <4.0.0' dependencies: - drift: ^2.15.0 + # drift: ^2.15.0 + drift: + git: + url: https://github.com/powersync-ja/drift.git + ref: test # branch name + path: drift sqlite3: '^2.3.0' js: ^0.6.3 async: ^2.10.0 From fc4b55d4e715fc4448c33f3e60673ff58de60844 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 6 Feb 2024 13:42:25 +0200 Subject: [PATCH 44/57] enable shared watch tests with custom Drift worker --- test/watch_shared_test.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/watch_shared_test.dart b/test/watch_shared_test.dart index 9155766..a19d3b7 100644 --- a/test/watch_shared_test.dart +++ b/test/watch_shared_test.dart @@ -1,5 +1,3 @@ -@TestOn('!browser') -// TODO watched queries require forked Drift lib worker import 'dart:async'; import 'dart:math'; import 'package:sqlite_async/sqlite_async.dart'; From f67d8022437069d9c8378f8372522de316f4ac3b Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 6 Feb 2024 16:59:18 +0200 Subject: [PATCH 45/57] migrate from to naming --- .github/workflows/test.yaml | 24 ++++++++++++------------ README.md | 2 +- lib/src/sqlite_options.dart | 4 ++-- lib/src/web/worker/drift_worker.dart | 2 +- test/server/worker_server.dart | 2 +- test/utils/web_test_utils.dart | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a41c669..0f78d22 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ name: Test on: push: branches: - - "**" + - '**' jobs: build: @@ -30,20 +30,20 @@ jobs: strategy: matrix: include: - - sqlite_version: "3440200" - sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" + - sqlite_version: '3440200' + sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz' dart_sdk: 3.2.4 - - sqlite_version: "3430200" - sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz" + - sqlite_version: '3430200' + sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz' dart_sdk: 3.2.4 - - sqlite_version: "3420000" - sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" + - sqlite_version: '3420000' + sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz' dart_sdk: 3.2.4 - - sqlite_version: "3410100" - sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" + - sqlite_version: '3410100' + sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz' dart_sdk: 3.2.4 - - sqlite_version: "3380000" - sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" + - sqlite_version: '3380000' + sqlite_url: 'https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz' dart_sdk: 3.2.0 steps: - uses: actions/checkout@v3 @@ -60,7 +60,7 @@ jobs: mkdir -p assets && curl -LJ https://github.com/simolus3/sqlite3.dart/releases/download/sqlite3-2.3.0/sqlite3.wasm -o assets/sqlite3.wasm - name: Compile WebWorker - run: dart compile js -o assets/drift_worker.js -O0 lib/src/web/worker/drift_worker.dart + run: dart compile js -o assets/db_worker.js -O0 lib/src/web/worker/drift_worker.dart - name: Run Tests run: | diff --git a/README.md b/README.md index 890fa64..583c1ba 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ final db = SqliteDatabase( path: 'test', options: SqliteOptions( webSqliteOptions: WebSqliteOptions( - wasmUri: 'sqlite3.wasm', workerUri: 'drift_worker.js'))); + wasmUri: 'sqlite3.wasm', workerUri: 'db_worker.js'))); ``` diff --git a/lib/src/sqlite_options.dart b/lib/src/sqlite_options.dart index 3bf4f15..6da7255 100644 --- a/lib/src/sqlite_options.dart +++ b/lib/src/sqlite_options.dart @@ -3,11 +3,11 @@ class WebSqliteOptions { final String wasmUri; const WebSqliteOptions.defaults() - : workerUri = 'drift_worker.js', + : workerUri = 'db_worker.js', wasmUri = 'sqlite3.wasm'; const WebSqliteOptions( - {this.wasmUri = 'sqlite3.wasm', this.workerUri = 'drift_worker.js'}); + {this.wasmUri = 'sqlite3.wasm', this.workerUri = 'db_worker.js'}); } class SqliteOptions { diff --git a/lib/src/web/worker/drift_worker.dart b/lib/src/web/worker/drift_worker.dart index 51860b7..4ef3956 100644 --- a/lib/src/web/worker/drift_worker.dart +++ b/lib/src/web/worker/drift_worker.dart @@ -1,7 +1,7 @@ /// This is an example of a database worker script /// Custom database logic can be achieved by implementing this template /// This file needs to be compiled to JavaScript with the command: -/// dart compile js -O4 lib/src/web/worker/drift_worker.dart -o build/drift_worker.js +/// dart compile js -O4 lib/src/web/worker/db_worker.dart -o build/db_worker.js /// The output should then be included in each project's `web` directory library; diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart index 53a3011..dee9b2b 100644 --- a/test/server/worker_server.dart +++ b/test/server/worker_server.dart @@ -20,7 +20,7 @@ Future hybridMain(StreamChannel channel) async { 'sqlite3.wasm file should be present in the ./assets folder'); } - final driftWorkerPath = p.join(directory.path, 'drift_worker.js'); + final driftWorkerPath = p.join(directory.path, 'db_worker.js'); if (!(await File(driftWorkerPath).exists())) { // And compile worker code final process = await Process.run(Platform.executable, [ diff --git a/test/utils/web_test_utils.dart b/test/utils/web_test_utils.dart index 9ba47ec..bbfe82d 100644 --- a/test/utils/web_test_utils.dart +++ b/test/utils/web_test_utils.dart @@ -22,7 +22,7 @@ class TestUtils extends AbstractTestUtils { final port = await channel.stream.first as int; final sqliteWasmUri = 'http://localhost:$port/sqlite3.wasm'; // Cross origin workers are not supported, but we can supply a Blob - var sqliteDriftUri = 'http://localhost:$port/drift_worker.js'; + var sqliteDriftUri = 'http://localhost:$port/db_worker.js'; final blob = Blob(['importScripts("$sqliteDriftUri");'], 'application/javascript'); From f946e1ad70d61ae0eee930c59e567e8a01f1ca77 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 7 Feb 2024 19:41:50 +0200 Subject: [PATCH 46/57] Use standard Drift package for now. Forked Drift is not published --- pubspec.yaml | 12 ++++++------ test/watch_shared_test.dart | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pubspec.yaml b/pubspec.yaml index a3fd7d2..0e11bb4 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,12 +6,12 @@ environment: sdk: '>=3.2.0 <4.0.0' dependencies: - # drift: ^2.15.0 - drift: - git: - url: https://github.com/powersync-ja/drift.git - ref: test # branch name - path: drift + drift: 2.15.0 + # drift: + # git: + # url: https://github.com/powersync-ja/drift.git + # ref: test # branch name + # path: drift sqlite3: '^2.3.0' js: ^0.6.3 async: ^2.10.0 diff --git a/test/watch_shared_test.dart b/test/watch_shared_test.dart index a19d3b7..f159a0f 100644 --- a/test/watch_shared_test.dart +++ b/test/watch_shared_test.dart @@ -1,3 +1,7 @@ +@TestOn('!browser') +// TODO watched query tests on web +// require a forked version of Drift's worker +// The forked version is not published yet. import 'dart:async'; import 'dart:math'; import 'package:sqlite_async/sqlite_async.dart'; From cd3386b16ae16cb5ad6586a894bed3a4128f0d2a Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 8 Feb 2024 10:30:54 +0200 Subject: [PATCH 47/57] less strict package version --- README.md | 7 ++++--- pubspec.yaml | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 583c1ba..bed5b26 100644 --- a/README.md +++ b/README.md @@ -78,13 +78,14 @@ void main() async { # Web -Web support is provided by the [Drift](https://drift.simonbinder.eu/web/) library. +Web support is provided by the [Drift](https://github.com/powersync-ja/drift/pull/1) library. Detailed instructions for compatibility and setup are listed in the link. Web support requires Sqlite3 WASM and Drift worker Javascript files to be accessible via configurable URIs. Default URIs are shown in the example below. URIs only need to be specified if they differ from default values. -Watched queries and table change notifications are only supported when using a custom Drift worker. [TBD release link] +Watched queries and table change notifications are only supported when using a custom Drift worker which is compiled by linking +https://github.com/powersync-ja/drift/pull/1. Setup @@ -92,7 +93,7 @@ Setup import 'package:sqlite_async/sqlite_async.dart'; final db = SqliteDatabase( - path: 'test', + path: 'test.db', options: SqliteOptions( webSqliteOptions: WebSqliteOptions( wasmUri: 'sqlite3.wasm', workerUri: 'db_worker.js'))); diff --git a/pubspec.yaml b/pubspec.yaml index 0e11bb4..d5be310 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -6,7 +6,8 @@ environment: sdk: '>=3.2.0 <4.0.0' dependencies: - drift: 2.15.0 + drift: ^2.15.0 + # In order to compile custom web worker # drift: # git: # url: https://github.com/powersync-ja/drift.git From fdd09c34475dbd5aa6681c4d3af1dadb5c9732b8 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Thu, 8 Feb 2024 10:42:35 +0200 Subject: [PATCH 48/57] code cleanup --- .github/workflows/test.yaml | 22 +++++++++---------- .../connection/sync_sqlite_connection.dart | 4 ++-- lib/src/native/database/connection_pool.dart | 13 +++++------ .../native_sqlite_connection_impl.dart | 11 +++++----- .../database/native_sqlite_database.dart | 20 ++++++++--------- .../native_isolate_connection_factory.dart | 10 ++++----- 6 files changed, 39 insertions(+), 41 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0f78d22..6a944d3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,7 +3,7 @@ name: Test on: push: branches: - - '**' + - "**" jobs: build: @@ -30,20 +30,20 @@ jobs: strategy: matrix: include: - - sqlite_version: '3440200' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz' + - sqlite_version: "3440200" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3430200' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz' + - sqlite_version: "3430200" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3430200.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3420000' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz' + - sqlite_version: "3420000" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3410100' - sqlite_url: 'https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz' + - sqlite_version: "3410100" + sqlite_url: "https://www.sqlite.org/2023/sqlite-autoconf-3410100.tar.gz" dart_sdk: 3.2.4 - - sqlite_version: '3380000' - sqlite_url: 'https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz' + - sqlite_version: "3380000" + sqlite_url: "https://www.sqlite.org/2022/sqlite-autoconf-3380000.tar.gz" dart_sdk: 3.2.0 steps: - uses: actions/checkout@v3 diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart index 648fa9a..a126821 100644 --- a/lib/src/common/connection/sync_sqlite_connection.dart +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -4,8 +4,8 @@ import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; -/// A simple "synchronous" connection which provides the SqliteConnection -/// implementation using a synchronous connection +/// A simple "synchronous" connection which provides the async SqliteConnection +/// implementation using a synchronous SQLite connection class SyncSqliteConnection extends SqliteConnection with SqliteQueries { final CommonDatabase db; late AbstractMutex mutex; diff --git a/lib/src/native/database/connection_pool.dart b/lib/src/native/database/connection_pool.dart index 7777d1b..bba4fd1 100644 --- a/lib/src/native/database/connection_pool.dart +++ b/lib/src/native/database/connection_pool.dart @@ -1,14 +1,13 @@ import 'dart:async'; import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; +import 'package:sqlite_async/src/native/database/native_sqlite_connection_impl.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; - -import '../../sqlite_connection.dart'; -import '../../sqlite_queries.dart'; -import '../../update_notification.dart'; -import '../../common/port_channel.dart'; -import 'native_sqlite_connection_impl.dart'; -import '../native_sqlite_open_factory.dart'; +import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; /// A connection pool with a single write connection and multiple read connections. class SqliteConnectionPool with SqliteQueries implements SqliteConnection { diff --git a/lib/src/native/database/native_sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart index 5f18a21..83b647d 100644 --- a/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -4,14 +4,13 @@ import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; - -import '../../utils/database_utils.dart'; -import '../../common/port_channel.dart'; -import '../../sqlite_connection.dart'; -import '../../sqlite_queries.dart'; -import '../../update_notification.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; typedef TxCallback = Future Function(CommonDatabase db); diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index 60cddf9..4b7ab40 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -1,18 +1,18 @@ import 'dart:async'; import 'dart:isolate'; +import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; +import 'package:sqlite_async/src/native/database/connection_pool.dart'; +import 'package:sqlite_async/src/native/database/native_sqlite_connection_impl.dart'; +import 'package:sqlite_async/src/native/native_isolate_connection_factory.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; - -import '../../utils/database_utils.dart'; -import '../../sqlite_connection.dart'; -import '../native_isolate_connection_factory.dart'; -import '../../sqlite_options.dart'; -import '../../update_notification.dart'; -import '../../common/abstract_sqlite_database.dart'; -import '../../common/port_channel.dart'; -import 'connection_pool.dart'; -import 'native_sqlite_connection_impl.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/native_database_utils.dart'; +import 'package:sqlite_async/src/utils/shared_utils.dart'; /// A SQLite database instance. /// diff --git a/lib/src/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart index d93b125..9a2cb31 100644 --- a/lib/src/native/native_isolate_connection_factory.dart +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -1,14 +1,14 @@ import 'dart:async'; import 'dart:isolate'; +import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; -import '../sqlite_connection.dart'; -import '../update_notification.dart'; -import '../utils/native_database_utils.dart'; -import '../common/port_channel.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/utils/database_utils.dart'; import 'database/native_sqlite_connection_impl.dart'; -import '../common/abstract_isolate_connection_factory.dart'; /// A connection factory that can be passed to different isolates. class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { From ba1622c5197c21005289902e14d7ffe91c2e43de Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 13 Feb 2024 12:18:47 +0200 Subject: [PATCH 49/57] improved auto commit check to only conditionally run --- lib/src/web/database/web_db_context.dart | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/src/web/database/web_db_context.dart b/lib/src/web/database/web_db_context.dart index bd89c90..6fd3358 100644 --- a/lib/src/web/database/web_db_context.dart +++ b/lib/src/web/database/web_db_context.dart @@ -80,10 +80,11 @@ class WebWriteContext extends WebReadContext implements SqliteWriteContext { if (_closed) { throw SqliteException(0, 'Transaction closed', null, sql); } - final isAutoCommit = await getAutoCommit(); /// Statements in read/writeTransactions should not execute after ROLLBACK - if (isTransaction && !sql.toLowerCase().contains('begin') && isAutoCommit) { + if (isTransaction && + !sql.toLowerCase().contains('begin') && + await getAutoCommit()) { throw SqliteException(0, 'Transaction rolled back by earlier statement. Cannot execute: $sql'); } From af05d85f364c3f9f2875dc53d1b2352c19449678 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 13 Feb 2024 15:29:09 +0200 Subject: [PATCH 50/57] remove some abstracted classes - replace with factory generators --- lib/definitions.dart | 8 ---- lib/sqlite_async.dart | 12 +++++- .../connection/sync_sqlite_connection.dart | 6 +-- ...y.dart => isolate_connection_factory.dart} | 43 ++++++++++++------- .../{abstract_mutex.dart => mutex.dart} | 4 +- ...ite_database.dart => sqlite_database.dart} | 41 +++++++++++++----- .../impl/stub_isolate_connection_factory.dart | 22 ++++++---- lib/src/impl/stub_mutex.dart | 4 +- lib/src/impl/stub_sqlite_database.dart | 19 ++++---- lib/src/mutex.dart | 2 +- lib/src/native/database/connection_pool.dart | 8 ++-- .../native_sqlite_connection_impl.dart | 6 +-- .../database/native_sqlite_database.dart | 24 ++++++----- .../native_isolate_connection_factory.dart | 12 +++--- lib/src/native/native_isolate_mutex.dart | 10 ++--- .../database/web_sqlite_connection_impl.dart | 5 +-- lib/src/web/database/web_sqlite_database.dart | 38 ++++++++-------- .../web/web_isolate_connection_factory.dart | 13 ++++-- lib/src/web/web_mutex.dart | 6 +-- scripts/benchmark.dart | 24 +++++------ test/basic_native_test.dart | 1 - test/basic_shared_test.dart | 1 - test/close_test.dart | 4 +- test/json1_test.dart | 2 +- test/watch_native_test.dart | 4 +- test/watch_shared_test.dart | 2 +- 26 files changed, 186 insertions(+), 135 deletions(-) delete mode 100644 lib/definitions.dart rename lib/src/common/{abstract_isolate_connection_factory.dart => isolate_connection_factory.dart} (59%) rename lib/src/common/{abstract_mutex.dart => mutex.dart} (91%) rename lib/src/common/{abstract_sqlite_database.dart => sqlite_database.dart} (62%) diff --git a/lib/definitions.dart b/lib/definitions.dart deleted file mode 100644 index 9781377..0000000 --- a/lib/definitions.dart +++ /dev/null @@ -1,8 +0,0 @@ -export 'package:sqlite_async/src/update_notification.dart'; -export 'package:sqlite_async/src/sqlite_connection.dart'; -export 'package:sqlite_async/src/sqlite_queries.dart'; -export 'package:sqlite_async/src/sqlite_open_factory.dart'; -export 'package:sqlite_async/src/sqlite_options.dart'; -export 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; -export 'package:sqlite_async/src/common/abstract_open_factory.dart'; -export 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index ec5f2bc..c843a06 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -3,13 +3,21 @@ /// See [SqliteDatabase] as a starting point. library; +export 'src/common/connection/sync_sqlite_connection.dart'; +export 'src/common/isolate_connection_factory.dart'; +export 'src/common/mutex.dart'; +export 'src/common/abstract_open_factory.dart'; +export 'src/common/port_channel.dart'; +export 'src/common/sqlite_database.dart'; +export 'src/impl/isolate_connection_factory_impl.dart'; +export 'src/impl/mutex_impl.dart'; +export 'src/impl/sqlite_database_impl.dart'; export 'src/isolate_connection_factory.dart'; export 'src/sqlite_connection.dart'; export 'src/sqlite_database.dart'; export 'src/sqlite_migrations.dart'; export 'src/sqlite_open_factory.dart'; +export 'src/sqlite_options.dart'; export 'src/sqlite_queries.dart'; export 'src/update_notification.dart'; export 'src/utils.dart'; -export 'definitions.dart'; -export 'src/common/connection/sync_sqlite_connection.dart'; diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart index a126821..5d1d339 100644 --- a/lib/src/common/connection/sync_sqlite_connection.dart +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -1,5 +1,5 @@ import 'package:sqlite3/common.dart'; -import 'package:sqlite_async/src/mutex.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; @@ -8,13 +8,13 @@ import 'package:sqlite_async/src/update_notification.dart'; /// implementation using a synchronous SQLite connection class SyncSqliteConnection extends SqliteConnection with SqliteQueries { final CommonDatabase db; - late AbstractMutex mutex; + late Mutex mutex; @override late final Stream updates; bool _closed = false; - SyncSqliteConnection(this.db, AbstractMutex m) { + SyncSqliteConnection(this.db, Mutex m) { mutex = m.open(); updates = db.updates.map( (event) { diff --git a/lib/src/common/abstract_isolate_connection_factory.dart b/lib/src/common/isolate_connection_factory.dart similarity index 59% rename from lib/src/common/abstract_isolate_connection_factory.dart rename to lib/src/common/isolate_connection_factory.dart index c3903e5..7a514d9 100644 --- a/lib/src/common/abstract_isolate_connection_factory.dart +++ b/lib/src/common/isolate_connection_factory.dart @@ -1,25 +1,14 @@ import 'dart:async'; -import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite3_common.dart' as sqlite; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/impl/isolate_connection_factory_impl.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; - -import 'abstract_open_factory.dart'; import 'port_channel.dart'; -/// A connection factory that can be passed to different isolates. -abstract class AbstractIsolateConnectionFactory< - Database extends sqlite.CommonDatabase> { +mixin IsolateOpenFactoryMixin { AbstractDefaultSqliteOpenFactory get openFactory; - AbstractMutex get mutex; - - SerializedPortClient get upstreamPort; - - /// Open a new SqliteConnection. - /// - /// This opens a single connection in a background execution isolate. - SqliteConnection open({String? debugName, bool readOnly = false}); - /// Opens a synchronous sqlite.Database directly in the current isolate. /// /// This gives direct access to the database, but: @@ -32,3 +21,27 @@ abstract class AbstractIsolateConnectionFactory< .open(SqliteOpenOptions(primaryConnection: false, readOnly: readOnly)); } } + +/// A connection factory that can be passed to different isolates. +abstract class IsolateConnectionFactory + with IsolateOpenFactoryMixin { + Mutex get mutex; + + SerializedPortClient get upstreamPort; + + factory IsolateConnectionFactory( + {required openFactory, + required mutex, + SerializedPortClient? upstreamPort}) { + return IsolateConnectionFactoryImpl( + openFactory: openFactory, + mutex: mutex, + upstreamPort: upstreamPort as SerializedPortClient) + as IsolateConnectionFactory; + } + + /// Open a new SqliteConnection. + /// + /// This opens a single connection in a background execution isolate. + SqliteConnection open({String? debugName, bool readOnly = false}); +} diff --git a/lib/src/common/abstract_mutex.dart b/lib/src/common/mutex.dart similarity index 91% rename from lib/src/common/abstract_mutex.dart rename to lib/src/common/mutex.dart index 3878991..dd82dcc 100644 --- a/lib/src/common/abstract_mutex.dart +++ b/lib/src/common/mutex.dart @@ -1,10 +1,10 @@ -abstract class AbstractMutex { +abstract class Mutex { /// timeout is a timeout for acquiring the lock, not for the callback Future lock(Future Function() callback, {Duration? timeout}); /// Use [open] to get a [AbstractMutex] instance. /// This is mainly used for shared mutexes - AbstractMutex open() { + Mutex open() { return this; } diff --git a/lib/src/common/abstract_sqlite_database.dart b/lib/src/common/sqlite_database.dart similarity index 62% rename from lib/src/common/abstract_sqlite_database.dart rename to lib/src/common/sqlite_database.dart index fbeaa84..30edd9a 100644 --- a/lib/src/common/abstract_sqlite_database.dart +++ b/lib/src/common/sqlite_database.dart @@ -1,20 +1,14 @@ import 'dart:async'; -import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/impl/sqlite_database_impl.dart'; +import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; -/// A SQLite database instance. -/// -/// Use one instance per database file. If multiple instances are used, update -/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. -abstract class AbstractSqliteDatabase extends SqliteConnection - with SqliteQueries { - /// The maximum number of concurrent read transactions if not explicitly specified. - static const int defaultMaxReaders = 5; - +mixin SqliteDatabaseMixin implements SqliteConnection, SqliteQueries { /// Maximum number of concurrent read transactions. int get maxReaders; @@ -44,5 +38,30 @@ abstract class AbstractSqliteDatabase extends SqliteConnection /// A connection factory that can be passed to different isolates. /// /// Use this to access the database in background isolates. - AbstractIsolateConnectionFactory isolateConnectionFactory(); + IsolateConnectionFactory isolateConnectionFactory(); +} + +/// A SQLite database instance. +/// +/// Use one instance per database file. If multiple instances are used, update +/// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. +abstract class SqliteDatabase + with SqliteQueries, SqliteDatabaseMixin + implements SqliteConnection { + /// The maximum number of concurrent read transactions if not explicitly specified. + static const int defaultMaxReaders = 5; + + factory SqliteDatabase( + {required path, + int maxReaders = SqliteDatabase.defaultMaxReaders, + SqliteOptions options = const SqliteOptions.defaults()}) { + return SqliteDatabaseImpl( + path: path, maxReaders: maxReaders, options: options); + } + + factory SqliteDatabase.withFactory( + AbstractDefaultSqliteOpenFactory openFactory, + {int maxReaders = SqliteDatabase.defaultMaxReaders}) { + return SqliteDatabaseImpl.withFactory(openFactory, maxReaders: maxReaders); + } } diff --git a/lib/src/impl/stub_isolate_connection_factory.dart b/lib/src/impl/stub_isolate_connection_factory.dart index ffc6416..208b7b8 100644 --- a/lib/src/impl/stub_isolate_connection_factory.dart +++ b/lib/src/impl/stub_isolate_connection_factory.dart @@ -1,18 +1,22 @@ import 'dart:async'; import 'package:sqlite3/common.dart'; -import 'package:sqlite_async/definitions.dart'; -import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; /// A connection factory that can be passed to different isolates. -class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { +class IsolateConnectionFactoryImpl + implements IsolateConnectionFactory { @override - AbstractDefaultSqliteOpenFactory openFactory; + AbstractDefaultSqliteOpenFactory openFactory; - IsolateConnectionFactory({ - required this.openFactory, - }); + IsolateConnectionFactoryImpl( + {required this.openFactory, + required Mutex mutex, + SerializedPortClient? upstreamPort}); @override @@ -31,12 +35,12 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { /// 2. Other connections are not notified of any updates to tables made within /// this connection. @override - Future openRawDatabase({bool readOnly = false}) async { + Future openRawDatabase({bool readOnly = false}) async { throw UnimplementedError(); } @override - AbstractMutex get mutex => throw UnimplementedError(); + Mutex get mutex => throw UnimplementedError(); @override SerializedPortClient get upstreamPort => throw UnimplementedError(); diff --git a/lib/src/impl/stub_mutex.dart b/lib/src/impl/stub_mutex.dart index 96ae62f..da22dec 100644 --- a/lib/src/impl/stub_mutex.dart +++ b/lib/src/impl/stub_mutex.dart @@ -1,6 +1,6 @@ -import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; -class Mutex extends AbstractMutex { +class MutexImpl extends Mutex { @override Future close() { throw UnimplementedError(); diff --git a/lib/src/impl/stub_sqlite_database.dart b/lib/src/impl/stub_sqlite_database.dart index bb317f9..adc8bbd 100644 --- a/lib/src/impl/stub_sqlite_database.dart +++ b/lib/src/impl/stub_sqlite_database.dart @@ -1,11 +1,14 @@ -import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; -import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/common/sqlite_database.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; -class SqliteDatabase extends AbstractSqliteDatabase { +class SqliteDatabaseImpl + with SqliteQueries, SqliteDatabaseMixin + implements SqliteDatabase { @override bool get closed => throw UnimplementedError(); @@ -15,15 +18,15 @@ class SqliteDatabase extends AbstractSqliteDatabase { @override int maxReaders; - factory SqliteDatabase( + factory SqliteDatabaseImpl( {required path, - int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + int maxReaders = SqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { throw UnimplementedError(); } - SqliteDatabase.withFactory(this.openFactory, - {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + SqliteDatabaseImpl.withFactory(this.openFactory, + {this.maxReaders = SqliteDatabase.defaultMaxReaders}) { throw UnimplementedError(); } @@ -51,7 +54,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { } @override - AbstractIsolateConnectionFactory isolateConnectionFactory() { + IsolateConnectionFactory isolateConnectionFactory() { throw UnimplementedError(); } diff --git a/lib/src/mutex.dart b/lib/src/mutex.dart index 8baa5ea..0c973c7 100644 --- a/lib/src/mutex.dart +++ b/lib/src/mutex.dart @@ -1,2 +1,2 @@ export 'impl/mutex_impl.dart'; -export 'common/abstract_mutex.dart'; +export 'common/mutex.dart'; diff --git a/lib/src/native/database/connection_pool.dart b/lib/src/native/database/connection_pool.dart index bba4fd1..4f92f27 100644 --- a/lib/src/native/database/connection_pool.dart +++ b/lib/src/native/database/connection_pool.dart @@ -1,10 +1,10 @@ import 'dart:async'; -import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/native/database/native_sqlite_connection_impl.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; -import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; @@ -15,7 +15,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final List _readConnections = []; - final DefaultSqliteOpenFactory _factory; + final AbstractDefaultSqliteOpenFactory _factory; final SerializedPortClient _upstreamPort; @override @@ -25,7 +25,7 @@ class SqliteConnectionPool with SqliteQueries implements SqliteConnection { final String? debugName; - final Mutex mutex; + final MutexImpl mutex; @override bool closed = false; diff --git a/lib/src/native/database/native_sqlite_connection_impl.dart b/lib/src/native/database/native_sqlite_connection_impl.dart index 83b647d..9351330 100644 --- a/lib/src/native/database/native_sqlite_connection_impl.dart +++ b/lib/src/native/database/native_sqlite_connection_impl.dart @@ -4,9 +4,9 @@ import 'dart:isolate'; import 'package:sqlite3/sqlite3.dart' as sqlite; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; -import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; @@ -65,7 +65,7 @@ class SqliteConnectionImpl with SqliteQueries implements SqliteConnection { } } - Future _open(DefaultSqliteOpenFactory openFactory, + Future _open(AbstractDefaultSqliteOpenFactory openFactory, {required bool primary, required SerializedPortClient upstreamPort}) async { await _connectionMutex.lock(() async { @@ -369,7 +369,7 @@ class _SqliteConnectionParams { final SerializedPortClient port; final bool primary; - final DefaultSqliteOpenFactory openFactory; + final AbstractDefaultSqliteOpenFactory openFactory; _SqliteConnectionParams( {required this.openFactory, diff --git a/lib/src/native/database/native_sqlite_database.dart b/lib/src/native/database/native_sqlite_database.dart index 4b7ab40..fd258d2 100644 --- a/lib/src/native/database/native_sqlite_database.dart +++ b/lib/src/native/database/native_sqlite_database.dart @@ -1,7 +1,8 @@ import 'dart:async'; import 'dart:isolate'; -import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/sqlite_database.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/native/database/connection_pool.dart'; import 'package:sqlite_async/src/native/database/native_sqlite_connection_impl.dart'; @@ -10,6 +11,7 @@ import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/utils/native_database_utils.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; @@ -18,9 +20,11 @@ import 'package:sqlite_async/src/utils/shared_utils.dart'; /// /// Use one instance per database file. If multiple instances are used, update /// notifications may not trigger, and calls may fail with "SQLITE_BUSY" errors. -class SqliteDatabase extends AbstractSqliteDatabase { +class SqliteDatabaseImpl + with SqliteQueries, SqliteDatabaseMixin + implements SqliteDatabase { @override - final DefaultSqliteOpenFactory openFactory; + final AbstractDefaultSqliteOpenFactory openFactory; @override late Stream updates; @@ -49,13 +53,13 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// from the last committed write transaction. /// /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabase( + factory SqliteDatabaseImpl( {required path, - int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + int maxReaders = SqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { final factory = DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + return SqliteDatabaseImpl.withFactory(factory, maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -67,8 +71,8 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(this.openFactory, - {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + SqliteDatabaseImpl.withFactory(this.openFactory, + {this.maxReaders = SqliteDatabase.defaultMaxReaders}) { updates = updatesController.stream; _listenForEvents(); @@ -149,8 +153,8 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// /// Use this to access the database in background isolates. @override - IsolateConnectionFactory isolateConnectionFactory() { - return IsolateConnectionFactory( + IsolateConnectionFactoryImpl isolateConnectionFactory() { + return IsolateConnectionFactoryImpl( openFactory: openFactory, mutex: mutex.shared, upstreamPort: _eventsPort.client()); diff --git a/lib/src/native/native_isolate_connection_factory.dart b/lib/src/native/native_isolate_connection_factory.dart index 9a2cb31..d49044d 100644 --- a/lib/src/native/native_isolate_connection_factory.dart +++ b/lib/src/native/native_isolate_connection_factory.dart @@ -1,19 +1,21 @@ import 'dart:async'; import 'dart:isolate'; -import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/native/native_isolate_mutex.dart'; -import 'package:sqlite_async/src/native/native_sqlite_open_factory.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/utils/database_utils.dart'; import 'database/native_sqlite_connection_impl.dart'; /// A connection factory that can be passed to different isolates. -class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { +class IsolateConnectionFactoryImpl + with IsolateOpenFactoryMixin + implements IsolateConnectionFactory { @override - DefaultSqliteOpenFactory openFactory; + AbstractDefaultSqliteOpenFactory openFactory; @override SerializedMutex mutex; @@ -21,7 +23,7 @@ class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { @override SerializedPortClient upstreamPort; - IsolateConnectionFactory( + IsolateConnectionFactoryImpl( {required this.openFactory, required this.mutex, required this.upstreamPort}); diff --git a/lib/src/native/native_isolate_mutex.dart b/lib/src/native/native_isolate_mutex.dart index 11aab78..df9fe0f 100644 --- a/lib/src/native/native_isolate_mutex.dart +++ b/lib/src/native/native_isolate_mutex.dart @@ -3,11 +3,11 @@ // (MIT) import 'dart:async'; -import 'package:sqlite_async/src/common/abstract_mutex.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; -abstract class Mutex extends AbstractMutex { - factory Mutex() { +abstract class MutexImpl implements Mutex { + factory MutexImpl() { return SimpleMutex(); } } @@ -15,7 +15,7 @@ abstract class Mutex extends AbstractMutex { /// Mutex maintains a queue of Future-returning functions that /// are executed sequentially. /// The internal lock is not shared across Isolates by default. -class SimpleMutex implements Mutex { +class SimpleMutex implements MutexImpl { // Adapted from https://github.com/tekartik/synchronized.dart/blob/master/synchronized/lib/src/basic_lock.dart Future? last; @@ -109,7 +109,7 @@ class SimpleMutex implements Mutex { /// Use [open] to get a [SharedMutex] instance. /// /// Uses a [SendPort] to communicate with the source mutex. -class SerializedMutex extends AbstractMutex { +class SerializedMutex extends Mutex { final SerializedPortClient client; SerializedMutex(this.client); diff --git a/lib/src/web/database/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart index 041257b..2eec28f 100644 --- a/lib/src/web/database/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -1,13 +1,12 @@ import 'dart:async'; import 'package:meta/meta.dart'; -import 'package:sqlite_async/src/common/abstract_mutex.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/update_notification.dart'; import 'package:sqlite_async/src/utils/shared_utils.dart'; -import 'package:sqlite_async/src/web/web_mutex.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'executor/sqlite_executor.dart'; @@ -115,7 +114,7 @@ class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { isTransaction: true); } - /// The [Mutex] on individual connections do already error in recursive locks. + /// The mutex on individual connections do already error in recursive locks. /// /// We duplicate the same check here, to: /// 1. Also error when the recursive transaction is handled by a different diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index 3cc5596..1b6d0fb 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -1,16 +1,20 @@ import 'dart:async'; - -import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; -import 'package:sqlite_async/src/sqlite_connection.dart'; +import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; +import 'package:sqlite_async/src/sqlite_queries.dart'; import 'package:sqlite_async/src/web/web_isolate_connection_factory.dart'; -import 'package:sqlite_async/src/web/web_mutex.dart'; -import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; +import 'package:sqlite_async/src/common/sqlite_database.dart'; +import 'package:sqlite_async/src/sqlite_connection.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; import 'package:sqlite_async/src/update_notification.dart'; +import 'package:sqlite_async/src/web/web_mutex.dart'; +import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'web_sqlite_connection_impl.dart'; -class SqliteDatabase extends AbstractSqliteDatabase { +class SqliteDatabaseImpl + with SqliteQueries, SqliteDatabaseMixin + implements SqliteDatabase { @override bool get closed { return _connection.closed; @@ -26,10 +30,10 @@ class SqliteDatabase extends AbstractSqliteDatabase { late Future isInitialized; @override - DefaultSqliteOpenFactory openFactory; + AbstractDefaultSqliteOpenFactory openFactory; late final Mutex mutex; - late final IsolateConnectionFactory _isolateConnectionFactory; + late final IsolateConnectionFactoryImpl _isolateConnectionFactory; late final WebSqliteConnectionImpl _connection; /// Open a SqliteDatabase. @@ -42,13 +46,13 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// from the last committed write transaction. /// /// A maximum of [maxReaders] concurrent read transactions are allowed. - factory SqliteDatabase( + factory SqliteDatabaseImpl( {required path, - int maxReaders = AbstractSqliteDatabase.defaultMaxReaders, + int maxReaders = SqliteDatabase.defaultMaxReaders, SqliteOptions options = const SqliteOptions.defaults()}) { final factory = DefaultSqliteOpenFactory(path: path, sqliteOptions: options); - return SqliteDatabase.withFactory(factory, maxReaders: maxReaders); + return SqliteDatabaseImpl.withFactory(factory, maxReaders: maxReaders); } /// Advanced: Open a database with a specified factory. @@ -60,12 +64,12 @@ class SqliteDatabase extends AbstractSqliteDatabase { /// 2. Running additional per-connection PRAGMA statements on each connection. /// 3. Creating custom SQLite functions. /// 4. Creating temporary views or triggers. - SqliteDatabase.withFactory(this.openFactory, - {this.maxReaders = AbstractSqliteDatabase.defaultMaxReaders}) { + SqliteDatabaseImpl.withFactory(this.openFactory, + {this.maxReaders = SqliteDatabase.defaultMaxReaders}) { updates = updatesController.stream; - mutex = Mutex(); - _isolateConnectionFactory = - IsolateConnectionFactory(openFactory: openFactory, mutex: mutex); + mutex = MutexImpl(); + _isolateConnectionFactory = IsolateConnectionFactoryImpl( + openFactory: openFactory as DefaultSqliteOpenFactory, mutex: mutex); _connection = _isolateConnectionFactory.open(); isInitialized = _init(); } @@ -112,7 +116,7 @@ class SqliteDatabase extends AbstractSqliteDatabase { } @override - IsolateConnectionFactory isolateConnectionFactory() { + IsolateConnectionFactoryImpl isolateConnectionFactory() { return _isolateConnectionFactory; } diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index abe6bac..7df7d9b 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -1,22 +1,27 @@ import 'dart:async'; import 'package:sqlite_async/sqlite3_common.dart'; -import 'package:sqlite_async/src/common/abstract_isolate_connection_factory.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; +import 'package:sqlite_async/src/common/isolate_connection_factory.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; -import 'web_mutex.dart'; import 'database/web_sqlite_connection_impl.dart'; /// A connection factory that can be passed to different isolates. -class IsolateConnectionFactory extends AbstractIsolateConnectionFactory { +class IsolateConnectionFactoryImpl + with IsolateOpenFactoryMixin + implements IsolateConnectionFactory { @override DefaultSqliteOpenFactory openFactory; @override Mutex mutex; - IsolateConnectionFactory({required this.openFactory, required this.mutex}); + IsolateConnectionFactoryImpl( + {required this.openFactory, + required this.mutex, + SerializedPortClient? upstreamPort}); /// Open a new SqliteConnection. /// diff --git a/lib/src/web/web_mutex.dart b/lib/src/web/web_mutex.dart index 7c95762..32f5896 100644 --- a/lib/src/web/web_mutex.dart +++ b/lib/src/web/web_mutex.dart @@ -1,10 +1,10 @@ import 'package:mutex/mutex.dart' as mutex; -import 'package:sqlite_async/src/mutex.dart'; +import 'package:sqlite_async/src/common/mutex.dart'; -class Mutex extends AbstractMutex { +class MutexImpl extends Mutex { late final mutex.Mutex m; - Mutex() { + MutexImpl() { m = mutex.Mutex(); } diff --git a/scripts/benchmark.dart b/scripts/benchmark.dart index c420e5f..ac919cf 100644 --- a/scripts/benchmark.dart +++ b/scripts/benchmark.dart @@ -25,7 +25,7 @@ class SqliteBenchmark { List benchmarks = [ SqliteBenchmark('Insert: JSON1', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { for (var i = 0; i < parameters.length; i += 5000) { var sublist = parameters.sublist(i, min(parameters.length, i + 5000)); @@ -37,7 +37,7 @@ List benchmarks = [ }); }, maxBatchSize: 20000), SqliteBenchmark('Read: JSON1', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.readTransaction((tx) async { for (var i = 0; i < parameters.length; i += 10000) { var sublist = List.generate(10000, (index) => index); @@ -60,26 +60,26 @@ List benchmarks = [ }); }, maxBatchSize: 10000, enabled: true), SqliteBenchmark('Write lock', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { for (var _ in parameters) { await db.writeLock((tx) async {}); } }, maxBatchSize: 5000, enabled: false), SqliteBenchmark('Read lock', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { for (var _ in parameters) { await db.readLock((tx) async {}); } }, maxBatchSize: 5000, enabled: false), SqliteBenchmark('Insert: Direct', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { for (var params in parameters) { await db.execute( 'INSERT INTO customers(name, email) VALUES(?, ?)', params); } }, maxBatchSize: 500), SqliteBenchmark('Insert: writeTransaction', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { for (var params in parameters) { await tx.execute( @@ -110,7 +110,7 @@ List benchmarks = [ }); }, maxBatchSize: 2000), SqliteBenchmark('Insert: writeTransaction no await', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { for (var params in parameters) { tx.execute('INSERT INTO customers(name, email) VALUES(?, ?)', params); @@ -118,7 +118,7 @@ List benchmarks = [ }); }, maxBatchSize: 1000), SqliteBenchmark('Insert: computeWithDatabase', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.computeWithDatabase((db) async { for (var params in parameters) { db.execute('INSERT INTO customers(name, email) VALUES(?, ?)', params); @@ -126,7 +126,7 @@ List benchmarks = [ }); }), SqliteBenchmark('Insert: computeWithDatabase, prepared', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.computeWithDatabase((db) async { var stmt = db.prepare('INSERT INTO customers(name, email) VALUES(?, ?)'); try { @@ -139,14 +139,14 @@ List benchmarks = [ }); }), SqliteBenchmark('Insert: executeBatch', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.writeTransaction((tx) async { await tx.executeBatch( 'INSERT INTO customers(name, email) VALUES(?, ?)', parameters); }); }), SqliteBenchmark('Insert: computeWithDatabase, prepared x10', - (AbstractSqliteDatabase db, List> parameters) async { + (SqliteDatabase db, List> parameters) async { await db.computeWithDatabase((db) async { var stmt = db.prepare( 'INSERT INTO customers(name, email) VALUES (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?), (?, ?)'); @@ -167,7 +167,7 @@ void main() async { var parameters = List.generate( 20000, (index) => ['Test user $index', 'user$index@example.org']); - createTables(AbstractSqliteDatabase db) async { + createTables(SqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute('DROP TABLE IF EXISTS customers'); await tx.execute( diff --git a/test/basic_native_test.dart b/test/basic_native_test.dart index eee324c..0315ff9 100644 --- a/test/basic_native_test.dart +++ b/test/basic_native_test.dart @@ -3,7 +3,6 @@ import 'dart:async'; import 'dart:math'; import 'package:sqlite3/common.dart' as sqlite; -import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; diff --git a/test/basic_shared_test.dart b/test/basic_shared_test.dart index bd3e128..dd98f70 100644 --- a/test/basic_shared_test.dart +++ b/test/basic_shared_test.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'package:sqlite_async/mutex.dart'; import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/sqlite_async.dart'; import 'package:test/test.dart'; diff --git a/test/close_test.dart b/test/close_test.dart index d3c62dd..dcb3390 100644 --- a/test/close_test.dart +++ b/test/close_test.dart @@ -2,7 +2,7 @@ import 'dart:io'; import 'package:sqlite_async/sqlite_async.dart'; -import 'package:sqlite_async/src/common/abstract_sqlite_database.dart'; +import 'package:sqlite_async/src/common/sqlite_database.dart'; import 'package:test/test.dart'; import 'utils/test_utils_impl.dart'; @@ -22,7 +22,7 @@ void main() { await testUtils.cleanDb(path: path); }); - createTables(AbstractSqliteDatabase db) async { + createTables(SqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE test_data(id INTEGER PRIMARY KEY AUTOINCREMENT, description TEXT)'); diff --git a/test/json1_test.dart b/test/json1_test.dart index 7717d43..93e44cc 100644 --- a/test/json1_test.dart +++ b/test/json1_test.dart @@ -36,7 +36,7 @@ void main() { await testUtils.cleanDb(path: path); }); - createTables(AbstractSqliteDatabase db) async { + createTables(SqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE users(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, email TEXT)'); diff --git a/test/watch_native_test.dart b/test/watch_native_test.dart index fc546fa..727408f 100644 --- a/test/watch_native_test.dart +++ b/test/watch_native_test.dart @@ -13,7 +13,7 @@ import 'utils/test_utils_impl.dart'; final testUtils = TestUtils(); void main() { - createTables(AbstractSqliteDatabase db) async { + createTables(SqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); @@ -127,7 +127,7 @@ void main() { }); } -Future> inIsolateWatch(AbstractIsolateConnectionFactory factory, +Future> inIsolateWatch(IsolateConnectionFactory factory, int numberOfQueries, Duration throttleDuration) async { return await Isolate.run(() async { final db = factory.open(); diff --git a/test/watch_shared_test.dart b/test/watch_shared_test.dart index f159a0f..ae3e468 100644 --- a/test/watch_shared_test.dart +++ b/test/watch_shared_test.dart @@ -12,7 +12,7 @@ import 'utils/test_utils_impl.dart'; final testUtils = TestUtils(); -createTables(AbstractSqliteDatabase db) async { +createTables(SqliteDatabase db) async { await db.writeTransaction((tx) async { await tx.execute( 'CREATE TABLE assets(id INTEGER PRIMARY KEY AUTOINCREMENT, make TEXT, customer_id INTEGER)'); From a35fa438dbdf9e0fa0ba6b7a8416f9ff1adde1f5 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 13 Feb 2024 15:38:41 +0200 Subject: [PATCH 51/57] fix getOptional method to throw expections correctly --- lib/sqlite_async.dart | 2 +- lib/src/common/connection/sync_sqlite_connection.dart | 7 ++----- lib/src/web/database/web_db_context.dart | 7 ++----- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index c843a06..e3f20d0 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -3,10 +3,10 @@ /// See [SqliteDatabase] as a starting point. library; +export 'src/common/abstract_open_factory.dart'; export 'src/common/connection/sync_sqlite_connection.dart'; export 'src/common/isolate_connection_factory.dart'; export 'src/common/mutex.dart'; -export 'src/common/abstract_open_factory.dart'; export 'src/common/port_channel.dart'; export 'src/common/sqlite_database.dart'; export 'src/impl/isolate_connection_factory_impl.dart'; diff --git a/lib/src/common/connection/sync_sqlite_connection.dart b/lib/src/common/connection/sync_sqlite_connection.dart index 5d1d339..f29520f 100644 --- a/lib/src/common/connection/sync_sqlite_connection.dart +++ b/lib/src/common/connection/sync_sqlite_connection.dart @@ -77,11 +77,8 @@ class SyncReadContext implements SqliteReadContext { @override Future getOptional(String sql, [List parameters = const []]) async { - try { - return db.select(sql, parameters).first; - } catch (ex) { - return null; - } + final rows = await getAll(sql, parameters); + return rows.isEmpty ? null : rows.first; } @override diff --git a/lib/src/web/database/web_db_context.dart b/lib/src/web/database/web_db_context.dart index 6fd3358..1c6b47f 100644 --- a/lib/src/web/database/web_db_context.dart +++ b/lib/src/web/database/web_db_context.dart @@ -40,11 +40,8 @@ class WebReadContext implements SqliteReadContext { @override Future getOptional(String sql, [List parameters = const []]) async { - try { - return (await getAll(sql, parameters)).first; - } catch (ex) { - return null; - } + final rows = await getAll(sql, parameters); + return rows.isEmpty ? null : rows.first; } @override From ab00e64cf10b90ad79356316e6241ce03a7e1880 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 13 Feb 2024 15:44:31 +0200 Subject: [PATCH 52/57] cleanup exports --- lib/sqlite_async.dart | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/sqlite_async.dart b/lib/sqlite_async.dart index e3f20d0..f930e5b 100644 --- a/lib/sqlite_async.dart +++ b/lib/sqlite_async.dart @@ -9,9 +9,6 @@ export 'src/common/isolate_connection_factory.dart'; export 'src/common/mutex.dart'; export 'src/common/port_channel.dart'; export 'src/common/sqlite_database.dart'; -export 'src/impl/isolate_connection_factory_impl.dart'; -export 'src/impl/mutex_impl.dart'; -export 'src/impl/sqlite_database_impl.dart'; export 'src/isolate_connection_factory.dart'; export 'src/sqlite_connection.dart'; export 'src/sqlite_database.dart'; From 0263cee98eb03e59c842b8e67e8e9f0eacde5e8f Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 13 Feb 2024 17:15:38 +0200 Subject: [PATCH 53/57] allow creating Mutex class from factory --- lib/src/common/mutex.dart | 6 ++++++ lib/src/impl/stub_mutex.dart | 7 ++++++- lib/src/native/native_isolate_mutex.dart | 2 +- lib/src/web/web_mutex.dart | 7 ++++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/src/common/mutex.dart b/lib/src/common/mutex.dart index dd82dcc..ccdcc49 100644 --- a/lib/src/common/mutex.dart +++ b/lib/src/common/mutex.dart @@ -1,4 +1,10 @@ +import 'package:sqlite_async/src/impl/mutex_impl.dart'; + abstract class Mutex { + factory Mutex() { + return MutexImpl(); + } + /// timeout is a timeout for acquiring the lock, not for the callback Future lock(Future Function() callback, {Duration? timeout}); diff --git a/lib/src/impl/stub_mutex.dart b/lib/src/impl/stub_mutex.dart index da22dec..1e700fa 100644 --- a/lib/src/impl/stub_mutex.dart +++ b/lib/src/impl/stub_mutex.dart @@ -1,6 +1,6 @@ import 'package:sqlite_async/src/common/mutex.dart'; -class MutexImpl extends Mutex { +class MutexImpl implements Mutex { @override Future close() { throw UnimplementedError(); @@ -10,4 +10,9 @@ class MutexImpl extends Mutex { Future lock(Future Function() callback, {Duration? timeout}) { throw UnimplementedError(); } + + @override + Mutex open() { + throw UnimplementedError(); + } } diff --git a/lib/src/native/native_isolate_mutex.dart b/lib/src/native/native_isolate_mutex.dart index df9fe0f..769f4da 100644 --- a/lib/src/native/native_isolate_mutex.dart +++ b/lib/src/native/native_isolate_mutex.dart @@ -109,7 +109,7 @@ class SimpleMutex implements MutexImpl { /// Use [open] to get a [SharedMutex] instance. /// /// Uses a [SendPort] to communicate with the source mutex. -class SerializedMutex extends Mutex { +class SerializedMutex implements Mutex { final SerializedPortClient client; SerializedMutex(this.client); diff --git a/lib/src/web/web_mutex.dart b/lib/src/web/web_mutex.dart index 32f5896..920cff5 100644 --- a/lib/src/web/web_mutex.dart +++ b/lib/src/web/web_mutex.dart @@ -1,7 +1,7 @@ import 'package:mutex/mutex.dart' as mutex; import 'package:sqlite_async/src/common/mutex.dart'; -class MutexImpl extends Mutex { +class MutexImpl implements Mutex { late final mutex.Mutex m; MutexImpl() { @@ -18,4 +18,9 @@ class MutexImpl extends Mutex { // TODO: use web navigator locks here return m.protect(callback); } + + @override + Mutex open() { + return this; + } } From 3960604ebbebd156e04965d765bb76018b973913 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Tue, 13 Feb 2024 17:33:46 +0200 Subject: [PATCH 54/57] neaten up comments --- lib/src/common/sqlite_database.dart | 19 +++++++++++++++++++ .../native/native_sqlite_open_factory.dart | 1 + .../database/web_sqlite_connection_impl.dart | 1 + lib/src/web/database/web_sqlite_database.dart | 2 ++ .../web/web_isolate_connection_factory.dart | 3 ++- lib/src/web/web_mutex.dart | 2 ++ lib/src/web/web_sqlite_open_factory.dart | 1 + test/server/worker_server.dart | 2 -- 8 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/src/common/sqlite_database.dart b/lib/src/common/sqlite_database.dart index 30edd9a..b8380c8 100644 --- a/lib/src/common/sqlite_database.dart +++ b/lib/src/common/sqlite_database.dart @@ -51,6 +51,16 @@ abstract class SqliteDatabase /// The maximum number of concurrent read transactions if not explicitly specified. static const int defaultMaxReaders = 5; + /// Open a SqliteDatabase. + /// + /// Only a single SqliteDatabase per [path] should be opened at a time. + /// + /// A connection pool is used by default, allowing multiple concurrent read + /// transactions, and a single concurrent write transaction. Write transactions + /// do not block read transactions, and read transactions will see the state + /// from the last committed write transaction. + /// + /// A maximum of [maxReaders] concurrent read transactions are allowed. factory SqliteDatabase( {required path, int maxReaders = SqliteDatabase.defaultMaxReaders, @@ -59,6 +69,15 @@ abstract class SqliteDatabase path: path, maxReaders: maxReaders, options: options); } + /// Advanced: Open a database with a specified factory. + /// + /// The factory is used to open each database connection in background isolates. + /// + /// Use when control is required over the opening process. Examples include: + /// 1. Specifying the path to `libsqlite.so` on Linux. + /// 2. Running additional per-connection PRAGMA statements on each connection. + /// 3. Creating custom SQLite functions. + /// 4. Creating temporary views or triggers. factory SqliteDatabase.withFactory( AbstractDefaultSqliteOpenFactory openFactory, {int maxReaders = SqliteDatabase.defaultMaxReaders}) { diff --git a/lib/src/native/native_sqlite_open_factory.dart b/lib/src/native/native_sqlite_open_factory.dart index 19fc700..c243bd0 100644 --- a/lib/src/native/native_sqlite_open_factory.dart +++ b/lib/src/native/native_sqlite_open_factory.dart @@ -4,6 +4,7 @@ import 'package:sqlite_async/sqlite3_common.dart'; import 'package:sqlite_async/src/common/abstract_open_factory.dart'; import 'package:sqlite_async/src/sqlite_options.dart'; +/// Native implementation of [AbstractDefaultSqliteOpenFactory] class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { const DefaultSqliteOpenFactory( {required super.path, diff --git a/lib/src/web/database/web_sqlite_connection_impl.dart b/lib/src/web/database/web_sqlite_connection_impl.dart index 2eec28f..a9df6b2 100644 --- a/lib/src/web/database/web_sqlite_connection_impl.dart +++ b/lib/src/web/database/web_sqlite_connection_impl.dart @@ -12,6 +12,7 @@ import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'executor/sqlite_executor.dart'; import 'web_db_context.dart'; +/// Web implementation of [SqliteConnection] class WebSqliteConnectionImpl with SqliteQueries implements SqliteConnection { @override bool get closed { diff --git a/lib/src/web/database/web_sqlite_database.dart b/lib/src/web/database/web_sqlite_database.dart index 1b6d0fb..62e3911 100644 --- a/lib/src/web/database/web_sqlite_database.dart +++ b/lib/src/web/database/web_sqlite_database.dart @@ -12,6 +12,8 @@ import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'web_sqlite_connection_impl.dart'; +/// Web implementation of [SqliteDatabase] +/// Uses a web worker for SQLite connection class SqliteDatabaseImpl with SqliteQueries, SqliteDatabaseMixin implements SqliteDatabase { diff --git a/lib/src/web/web_isolate_connection_factory.dart b/lib/src/web/web_isolate_connection_factory.dart index 7df7d9b..68212c2 100644 --- a/lib/src/web/web_isolate_connection_factory.dart +++ b/lib/src/web/web_isolate_connection_factory.dart @@ -8,7 +8,8 @@ import 'package:sqlite_async/src/common/port_channel.dart'; import 'package:sqlite_async/src/web/web_sqlite_open_factory.dart'; import 'database/web_sqlite_connection_impl.dart'; -/// A connection factory that can be passed to different isolates. +/// An implementation of [IsolateConnectionFactory] for Web +/// This uses a web worker instead of an isolate class IsolateConnectionFactoryImpl with IsolateOpenFactoryMixin implements IsolateConnectionFactory { diff --git a/lib/src/web/web_mutex.dart b/lib/src/web/web_mutex.dart index 920cff5..1a5752f 100644 --- a/lib/src/web/web_mutex.dart +++ b/lib/src/web/web_mutex.dart @@ -1,6 +1,8 @@ import 'package:mutex/mutex.dart' as mutex; import 'package:sqlite_async/src/common/mutex.dart'; +/// Web implementation of [Mutex] +/// This will use `navigator.locks` in future class MutexImpl implements Mutex { late final mutex.Mutex m; diff --git a/lib/src/web/web_sqlite_open_factory.dart b/lib/src/web/web_sqlite_open_factory.dart index fa73071..916d275 100644 --- a/lib/src/web/web_sqlite_open_factory.dart +++ b/lib/src/web/web_sqlite_open_factory.dart @@ -9,6 +9,7 @@ import 'package:sqlite_async/src/sqlite_options.dart'; import 'database/executor/drift_sql_executor.dart'; import 'database/executor/sqlite_executor.dart'; +/// Web implementation of [AbstractDefaultSqliteOpenFactory] class DefaultSqliteOpenFactory extends AbstractDefaultSqliteOpenFactory { DefaultSqliteOpenFactory( diff --git a/test/server/worker_server.dart b/test/server/worker_server.dart index dee9b2b..394ff89 100644 --- a/test/server/worker_server.dart +++ b/test/server/worker_server.dart @@ -12,7 +12,6 @@ import 'asset_server.dart'; Future hybridMain(StreamChannel channel) async { final directory = Directory('./assets'); - // Copy sqlite3.wasm file expected by the worker final sqliteOutputPath = p.join(directory.path, 'sqlite3.wasm'); if (!(await File(sqliteOutputPath).exists())) { @@ -22,7 +21,6 @@ Future hybridMain(StreamChannel channel) async { final driftWorkerPath = p.join(directory.path, 'db_worker.js'); if (!(await File(driftWorkerPath).exists())) { - // And compile worker code final process = await Process.run(Platform.executable, [ 'compile', 'js', From 9adb4c603fc90c0d5150d5b431299cee509ec61a Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 14 Feb 2024 10:18:28 +0200 Subject: [PATCH 55/57] temporarily bump exit-code-threshold to 10. While waiting for dependnecy to be updated in sqlite3.dart --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6a944d3..b226d7e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,7 @@ jobs: - name: Check publish score run: | dart pub global activate pana - dart pub global run pana --no-warning --exit-code-threshold 0 + dart pub global run pana --no-warning --exit-code-threshold 10 test: runs-on: ubuntu-latest From 22752668b88223a7f46a86cf0664159ffb175789 Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 14 Feb 2024 10:39:26 +0200 Subject: [PATCH 56/57] bump package version --- pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubspec.yaml b/pubspec.yaml index d5be310..bbce796 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlite_async description: High-performance asynchronous interface for SQLite on Dart and Flutter. -version: 0.6.0 +version: 0.7.0-alpha.1 repository: https://github.com/powersync-ja/sqlite_async.dart environment: sdk: '>=3.2.0 <4.0.0' From 99d6289d1ccf1804240741137000d4b8ddb6104d Mon Sep 17 00:00:00 2001 From: Steven Ontong Date: Wed, 14 Feb 2024 10:41:24 +0200 Subject: [PATCH 57/57] added changelog entry --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb3af42..f895b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.7.0-alpha.1 + +- Added initial support for web platform. + ## 0.6.0 - Allow catching errors and continuing the transaction. This is technically a breaking change, although it should not be an issue in most cases.